Quelle  video_stream_encoder.cc   Sprache: C

/*
 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "video/video_stream_encoder.h"

#include <algorithm>
#include <array>
#include <limits>
#include <memory>
#include <numeric>
#include <optional>
#include <utility>

#include "absl/algorithm/container.h"
#include "absl/cleanup/cleanup.h"
#include "absl/types/variant.h"
#include "api/field_trials_view.h"
#include "api/sequence_checker.h"
#include "api/task_queue/task_queue_base.h"
#include "api/video/encoded_image.h"
#include "api/video/i420_buffer.h"
#include "api/video/render_resolution.h"
#include "api/video/video_adaptation_reason.h"
#include "api/video/video_bitrate_allocator_factory.h"
#include "api/video/video_codec_constants.h"
#include "api/video/video_layers_allocation.h"
#include "api/video/video_stream_encoder_settings.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_encoder.h"
#include "call/adaptation/resource_adaptation_processor.h"
#include "call/adaptation/video_source_restrictions.h"
#include "call/adaptation/video_stream_adapter.h"
#include "common_video/frame_instrumentation_data.h"
#include "media/base/media_channel.h"
#include "modules/video_coding/include/video_codec_initializer.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "modules/video_coding/svc/scalability_mode_util.h"
#include "modules/video_coding/svc/svc_rate_allocator.h"
#include "modules/video_coding/utility/vp8_constants.h"
#include "rtc_base/arraysize.h"
#include "rtc_base/checks.h"
#include "rtc_base/event.h"
#include "rtc_base/experiments/encoder_info_settings.h"
#include "rtc_base/experiments/rate_control_settings.h"
#include "rtc_base/logging.h"
#include "rtc_base/strings/string_builder.h"
#include "rtc_base/system/no_unique_address.h"
#include "rtc_base/thread_annotations.h"
#include "rtc_base/trace_event.h"
#include "system_wrappers/include/metrics.h"
#include "video/adaptation/video_stream_encoder_resource_manager.h"
#include "video/alignment_adjuster.h"
#include "video/config/encoder_stream_factory.h"
#include "video/config/video_encoder_config.h"
#include "video/corruption_detection/frame_instrumentation_generator.h"
#include "video/frame_cadence_adapter.h"
#include "video/frame_dumping_encoder.h"

namespace webrtc {

namespace {

// Time interval for logging frame counts.
const int64_t kFrameLogIntervalMs = 60000;

// Time to keep a single cached pending frame in paused state.
const int64_t kPendingFrameTimeoutMs = 1000;

constexpr char kFrameDropperFieldTrial[] = "WebRTC-FrameDropper";

// TODO(bugs.webrtc.org/13572): Remove this kill switch after deploying the
// feature.
constexpr char kSwitchEncoderOnInitializationFailuresFieldTrial[] =
    "WebRTC-SwitchEncoderOnInitializationFailures";

const size_t kDefaultPayloadSize = 1440;

const int64_t kParameterUpdateIntervalMs = 1000;

constexpr int kDefaultMinScreenSharebps = 1200000;

int GetNumSpatialLayers(const VideoCodec& codec) {
  if (codec.codecType == kVideoCodecVP9) {
    return codec.VP9().numberOfSpatialLayers;
  } else if (codec.codecType == kVideoCodecAV1 &&
             codec.GetScalabilityMode().has_value()) {
    return ScalabilityModeToNumSpatialLayers(*(codec.GetScalabilityMode()));
  } else {
    return 0;
  }
}

std::optional<EncodedImageCallback::DropReason> MaybeConvertDropReason(
    VideoStreamEncoderObserver::DropReason reason) {
  switch (reason) {
    case VideoStreamEncoderObserver::DropReason::kMediaOptimization:
      return EncodedImageCallback::DropReason::kDroppedByMediaOptimizations;
    case VideoStreamEncoderObserver::DropReason::kEncoder:
      return EncodedImageCallback::DropReason::kDroppedByEncoder;
    default:
      return std::nullopt;
  }
}

bool RequiresEncoderReset(const VideoCodec& prev_send_codec,
                          const VideoCodec& new_send_codec,
                          bool was_encode_called_since_last_initialization) {
  // Does not check max/minBitrate or maxFramerate.
  if (new_send_codec.codecType != prev_send_codec.codecType ||
      new_send_codec.width != prev_send_codec.width ||
      new_send_codec.height != prev_send_codec.height ||
      new_send_codec.qpMax != prev_send_codec.qpMax ||
      new_send_codec.numberOfSimulcastStreams !=
          prev_send_codec.numberOfSimulcastStreams ||
      new_send_codec.mode != prev_send_codec.mode ||
      new_send_codec.GetFrameDropEnabled() !=
          prev_send_codec.GetFrameDropEnabled()) {
    return true;
  }

  if (!was_encode_called_since_last_initialization &&
      (new_send_codec.startBitrate != prev_send_codec.startBitrate)) {
    // If start bitrate has changed reconfigure encoder only if encoding had not
    // yet started.
    return true;
  }

  switch (new_send_codec.codecType) {
    case kVideoCodecVP8:
      if (new_send_codec.VP8() != prev_send_codec.VP8()) {
        return true;
      }
      break;

    case kVideoCodecVP9:
      if (new_send_codec.VP9() != prev_send_codec.VP9()) {
        return true;
      }
      break;

    case kVideoCodecH264:
      if (new_send_codec.H264() != prev_send_codec.H264()) {
        return true;
      }
      break;
    case kVideoCodecH265:
      // No H.265 specific handling needed.
      [[fallthrough]];
    default:
      break;
  }

  for (unsigned char i = 0; i < new_send_codec.numberOfSimulcastStreams; ++i) {
    if (!new_send_codec.simulcastStream[i].active) {
      // No need to reset when stream is inactive.
      continue;
    }

    if (!prev_send_codec.simulcastStream[i].active ||
        new_send_codec.simulcastStream[i].width !=
            prev_send_codec.simulcastStream[i].width ||
        new_send_codec.simulcastStream[i].height !=
            prev_send_codec.simulcastStream[i].height ||
        new_send_codec.simulcastStream[i].numberOfTemporalLayers !=
            prev_send_codec.simulcastStream[i].numberOfTemporalLayers ||
        new_send_codec.simulcastStream[i].qpMax !=
            prev_send_codec.simulcastStream[i].qpMax) {
      return true;
    }

    if (new_send_codec.simulcastStream[i].maxFramerate !=
            prev_send_codec.simulcastStream[i].maxFramerate &&
        new_send_codec.simulcastStream[i].maxFramerate !=
            new_send_codec.maxFramerate) {
      // SetRates can only represent maxFramerate for one layer. Reset the
      // encoder if there are multiple layers that differ in maxFramerate.
      return true;
    }
  }

  if (new_send_codec.codecType == kVideoCodecVP9) {
    size_t num_spatial_layers = new_send_codec.VP9().numberOfSpatialLayers;
    for (unsigned char i = 0; i < num_spatial_layers; ++i) {
      if (!new_send_codec.spatialLayers[i].active) {
        // No need to reset when layer is inactive.
        continue;
      }
      if (new_send_codec.spatialLayers[i].width !=
              prev_send_codec.spatialLayers[i].width ||
          new_send_codec.spatialLayers[i].height !=
              prev_send_codec.spatialLayers[i].height ||
          new_send_codec.spatialLayers[i].numberOfTemporalLayers !=
              prev_send_codec.spatialLayers[i].numberOfTemporalLayers ||
          new_send_codec.spatialLayers[i].qpMax !=
              prev_send_codec.spatialLayers[i].qpMax ||
          !prev_send_codec.spatialLayers[i].active) {
        return true;
      }
    }
  }

  if (new_send_codec.GetScalabilityMode() !=
      prev_send_codec.GetScalabilityMode()) {
    return true;
  }

  return false;
}

// Limit allocation across TLs in bitrate allocation according to number of TLs
// in EncoderInfo.
VideoBitrateAllocation UpdateAllocationFromEncoderInfo(
    const VideoBitrateAllocation& allocation,
    const VideoEncoder::EncoderInfo& encoder_info) {
  if (allocation.get_sum_bps() == 0) {
    return allocation;
  }
  VideoBitrateAllocation new_allocation;
  for (int si = 0; si < kMaxSpatialLayers; ++si) {
    if (encoder_info.fps_allocation[si].size() == 1 &&
        allocation.IsSpatialLayerUsed(si)) {
      // One TL is signalled to be used by the encoder. Do not distribute
      // bitrate allocation across TLs (use sum at ti:0).
      new_allocation.SetBitrate(si, 0, allocation.GetSpatialLayerSum(si));
    } else {
      for (int ti = 0; ti < kMaxTemporalStreams; ++ti) {
        if (allocation.HasBitrate(si, ti))
          new_allocation.SetBitrate(si, ti, allocation.GetBitrate(si, ti));
      }
    }
  }
  new_allocation.set_bw_limited(allocation.is_bw_limited());
  return new_allocation;
}

// Converts a VideoBitrateAllocation that contains allocated bitrate per layer,
// and an EncoderInfo that contains information about the actual encoder
// structure used by a codec. Stream structures can be Ksvc, Full SVC, Simulcast
// etc.
VideoLayersAllocation CreateVideoLayersAllocation(
    const VideoCodec& encoder_config,
    const VideoEncoder::RateControlParameters& current_rate,
    const VideoEncoder::EncoderInfo& encoder_info) {
  const VideoBitrateAllocation& target_bitrate = current_rate.target_bitrate;
  VideoLayersAllocation layers_allocation;
  if (target_bitrate.get_sum_bps() == 0) {
    return layers_allocation;
  }

  if (encoder_config.numberOfSimulcastStreams > 1) {
    layers_allocation.resolution_and_frame_rate_is_valid = true;
    for (int si = 0; si < encoder_config.numberOfSimulcastStreams; ++si) {
      if (!target_bitrate.IsSpatialLayerUsed(si) ||
          target_bitrate.GetSpatialLayerSum(si) == 0) {
        continue;
      }
      layers_allocation.active_spatial_layers.emplace_back();
      VideoLayersAllocation::SpatialLayer& spatial_layer =
          layers_allocation.active_spatial_layers.back();
      spatial_layer.width = encoder_config.simulcastStream[si].width;
      spatial_layer.height = encoder_config.simulcastStream[si].height;
      spatial_layer.rtp_stream_index = si;
      spatial_layer.spatial_id = 0;
      auto frame_rate_fraction =
          VideoEncoder::EncoderInfo::kMaxFramerateFraction;
      if (encoder_info.fps_allocation[si].size() == 1) {
        // One TL is signalled to be used by the encoder. Do not distribute
        // bitrate allocation across TLs (use sum at tl:0).
        spatial_layer.target_bitrate_per_temporal_layer.push_back(
            DataRate::BitsPerSec(target_bitrate.GetSpatialLayerSum(si)));
        frame_rate_fraction = encoder_info.fps_allocation[si][0];
      } else {  // Temporal layers are supported.
        uint32_t temporal_layer_bitrate_bps = 0;
        for (size_t ti = 0;
             ti < encoder_config.simulcastStream[si].numberOfTemporalLayers;
             ++ti) {
          if (!target_bitrate.HasBitrate(si, ti)) {
            break;
          }
          if (ti < encoder_info.fps_allocation[si].size()) {
            // Use frame rate of the top used temporal layer.
            frame_rate_fraction = encoder_info.fps_allocation[si][ti];
          }
          temporal_layer_bitrate_bps += target_bitrate.GetBitrate(si, ti);
          spatial_layer.target_bitrate_per_temporal_layer.push_back(
              DataRate::BitsPerSec(temporal_layer_bitrate_bps));
        }
      }
      // Encoder may drop frames internally if `maxFramerate` is set.
      spatial_layer.frame_rate_fps = std::min<uint8_t>(
          encoder_config.simulcastStream[si].maxFramerate,
          rtc::saturated_cast<uint8_t>(
              (current_rate.framerate_fps * frame_rate_fraction) /
              VideoEncoder::EncoderInfo::kMaxFramerateFraction));
    }
  } else if (encoder_config.numberOfSimulcastStreams == 1) {
    // TODO(bugs.webrtc.org/12000): Implement support for AV1 with
    // scalability.
    const bool higher_spatial_depend_on_lower =
        encoder_config.codecType == kVideoCodecVP9 &&
        encoder_config.VP9().interLayerPred == InterLayerPredMode::kOn;
    layers_allocation.resolution_and_frame_rate_is_valid = true;

    std::vector<DataRate> aggregated_spatial_bitrate(
        webrtc::kMaxTemporalStreams, DataRate::Zero());
    for (int si = 0; si < webrtc::kMaxSpatialLayers; ++si) {
      layers_allocation.resolution_and_frame_rate_is_valid = true;
      if (!target_bitrate.IsSpatialLayerUsed(si) ||
          target_bitrate.GetSpatialLayerSum(si) == 0) {
        break;
      }
      layers_allocation.active_spatial_layers.emplace_back();
      VideoLayersAllocation::SpatialLayer& spatial_layer =
          layers_allocation.active_spatial_layers.back();
      spatial_layer.width = encoder_config.spatialLayers[si].width;
      spatial_layer.height = encoder_config.spatialLayers[si].height;
      spatial_layer.rtp_stream_index = 0;
      spatial_layer.spatial_id = si;
      auto frame_rate_fraction =
          VideoEncoder::EncoderInfo::kMaxFramerateFraction;
      if (encoder_info.fps_allocation[si].size() == 1) {
        // One TL is signalled to be used by the encoder. Do not distribute
        // bitrate allocation across TLs (use sum at tl:0).
        DataRate aggregated_temporal_bitrate =
            DataRate::BitsPerSec(target_bitrate.GetSpatialLayerSum(si));
        aggregated_spatial_bitrate[0] += aggregated_temporal_bitrate;
        if (higher_spatial_depend_on_lower) {
          spatial_layer.target_bitrate_per_temporal_layer.push_back(
              aggregated_spatial_bitrate[0]);
        } else {
          spatial_layer.target_bitrate_per_temporal_layer.push_back(
              aggregated_temporal_bitrate);
        }
        frame_rate_fraction = encoder_info.fps_allocation[si][0];
      } else {  // Temporal layers are supported.
        DataRate aggregated_temporal_bitrate = DataRate::Zero();
        for (size_t ti = 0;
             ti < encoder_config.spatialLayers[si].numberOfTemporalLayers;
             ++ti) {
          if (!target_bitrate.HasBitrate(si, ti)) {
            break;
          }
          if (ti < encoder_info.fps_allocation[si].size()) {
            // Use frame rate of the top used temporal layer.
            frame_rate_fraction = encoder_info.fps_allocation[si][ti];
          }
          aggregated_temporal_bitrate +=
              DataRate::BitsPerSec(target_bitrate.GetBitrate(si, ti));
          if (higher_spatial_depend_on_lower) {
            spatial_layer.target_bitrate_per_temporal_layer.push_back(
                aggregated_temporal_bitrate + aggregated_spatial_bitrate[ti]);
            aggregated_spatial_bitrate[ti] += aggregated_temporal_bitrate;
          } else {
            spatial_layer.target_bitrate_per_temporal_layer.push_back(
                aggregated_temporal_bitrate);
          }
        }
      }
      // Encoder may drop frames internally if `maxFramerate` is set.
      spatial_layer.frame_rate_fps = std::min<uint8_t>(
          encoder_config.spatialLayers[si].maxFramerate,
          rtc::saturated_cast<uint8_t>(
              (current_rate.framerate_fps * frame_rate_fraction) /
              VideoEncoder::EncoderInfo::kMaxFramerateFraction));
    }
  }

  return layers_allocation;
}

VideoEncoder::EncoderInfo GetEncoderInfoWithBitrateLimitUpdate(
    const VideoEncoder::EncoderInfo& info,
    const VideoEncoderConfig& encoder_config,
    bool default_limits_allowed) {
  if (!default_limits_allowed || !info.resolution_bitrate_limits.empty() ||
      encoder_config.simulcast_layers.size() <= 1) {
    return info;
  }
  // Bitrate limits are not configured and more than one layer is used, use
  // the default limits (bitrate limits are not used for simulcast).
  VideoEncoder::EncoderInfo new_info = info;
  new_info.resolution_bitrate_limits =
      EncoderInfoSettings::GetDefaultSinglecastBitrateLimits(
          encoder_config.codec_type);
  return new_info;
}

int NumActiveStreams(const std::vector<VideoStream>& streams) {
  int num_active = 0;
  for (const auto& stream : streams) {
    if (stream.active)
      ++num_active;
  }
  return num_active;
}

void ApplySpatialLayerBitrateLimits(
    const VideoEncoder::EncoderInfo& encoder_info,
    const VideoEncoderConfig& encoder_config,
    VideoCodec* codec) {
  if (!(GetNumSpatialLayers(*codec) > 0)) {
    // ApplySpatialLayerBitrateLimits() supports VP9 and AV1 (the latter with
    // scalability mode set) only.
    return;
  }
  if (VideoStreamEncoderResourceManager::IsSimulcastOrMultipleSpatialLayers(
          encoder_config, *codec) ||
      encoder_config.simulcast_layers.size() <= 1) {
    // Resolution bitrate limits usage is restricted to singlecast.
    return;
  }

  // Get bitrate limits for active stream.
  std::optional<uint32_t> pixels =
      VideoStreamAdapter::GetSingleActiveLayerPixels(*codec);
  if (!pixels.has_value()) {
    return;
  }
  std::optional<VideoEncoder::ResolutionBitrateLimits> bitrate_limits =
      encoder_info.GetEncoderBitrateLimitsForResolution(*pixels);
  if (!bitrate_limits.has_value()) {
    return;
  }
  // Index for the active stream.
  std::optional<size_t> index;
  for (size_t i = 0; i < encoder_config.simulcast_layers.size(); ++i) {
    if (encoder_config.simulcast_layers[i].active)
      index = i;
  }
  if (!index.has_value()) {
    return;
  }
  int min_bitrate_bps;
  if (encoder_config.simulcast_layers[*index].min_bitrate_bps <= 0) {
    min_bitrate_bps = bitrate_limits->min_bitrate_bps;
  } else {
    min_bitrate_bps = encoder_config.simulcast_layers[*index].min_bitrate_bps;
  }
  int max_bitrate_bps;
  if (encoder_config.simulcast_layers[*index].max_bitrate_bps <= 0) {
    max_bitrate_bps = bitrate_limits->max_bitrate_bps;
  } else {
    max_bitrate_bps =
        std::min(bitrate_limits->max_bitrate_bps,
                 encoder_config.simulcast_layers[*index].max_bitrate_bps);
  }
  if (min_bitrate_bps >= max_bitrate_bps) {
    RTC_LOG(LS_WARNING) << "Bitrate limits not used, min_bitrate_bps "
                        << min_bitrate_bps << " >= max_bitrate_bps "
                        << max_bitrate_bps;
    return;
  }

  for (int i = 0; i < GetNumSpatialLayers(*codec); ++i) {
    if (codec->spatialLayers[i].active) {
      codec->spatialLayers[i].minBitrate = min_bitrate_bps / 1000;
      codec->spatialLayers[i].maxBitrate = max_bitrate_bps / 1000;
      codec->spatialLayers[i].targetBitrate =
          std::min(codec->spatialLayers[i].targetBitrate,
                   codec->spatialLayers[i].maxBitrate);
      break;
    }
  }
}

void ApplyEncoderBitrateLimitsIfSingleActiveStream(
    const VideoEncoder::EncoderInfo& encoder_info,
    const std::vector<VideoStream>& encoder_config_layers,
    std::vector<VideoStream>* streams) {
  // Apply limits if simulcast with one active stream (expect lowest).
  bool single_active_stream =
      streams->size() > 1 && NumActiveStreams(*streams) == 1 &&
      !streams->front().active && NumActiveStreams(encoder_config_layers) == 1;
  if (!single_active_stream) {
    return;
  }

  // Index for the active stream.
  size_t index = 0;
  for (size_t i = 0; i < encoder_config_layers.size(); ++i) {
    if (encoder_config_layers[i].active)
      index = i;
  }
  if (streams->size() < (index + 1) || !(*streams)[index].active) {
    return;
  }

  // Get bitrate limits for active stream.
  std::optional<VideoEncoder::ResolutionBitrateLimits> encoder_bitrate_limits =
      encoder_info.GetEncoderBitrateLimitsForResolution(
          (*streams)[index].width * (*streams)[index].height);
  if (!encoder_bitrate_limits) {
    return;
  }

  int min_bitrate_bps;
  if (encoder_config_layers[index].min_bitrate_bps <= 0) {
    min_bitrate_bps = encoder_bitrate_limits->min_bitrate_bps;
  } else {
    min_bitrate_bps = (*streams)[index].min_bitrate_bps;
  }
  int max_bitrate_bps;
  if (encoder_config_layers[index].max_bitrate_bps <= 0) {
    max_bitrate_bps = encoder_bitrate_limits->max_bitrate_bps;
  } else {
    max_bitrate_bps = std::min(encoder_bitrate_limits->max_bitrate_bps,
                               (*streams)[index].max_bitrate_bps);
  }
  if (min_bitrate_bps >= max_bitrate_bps) {
    RTC_LOG(LS_WARNING) << "Encoder bitrate limits"
                        << " (min=" << encoder_bitrate_limits->min_bitrate_bps
                        << ", max=" << encoder_bitrate_limits->max_bitrate_bps
                        << ") do not intersect with stream limits"
                        << " (min=" << (*streams)[index].min_bitrate_bps
                        << ", max=" << (*streams)[index].max_bitrate_bps
                        << "). Encoder bitrate limits not used.";
    return;
  }

  (*streams)[index].min_bitrate_bps = min_bitrate_bps;
  (*streams)[index].max_bitrate_bps = max_bitrate_bps;
  (*streams)[index].target_bitrate_bps =
      std::min((*streams)[index].target_bitrate_bps,
               encoder_bitrate_limits->max_bitrate_bps);
}

std::optional<int> ParseVp9LowTierCoreCountThreshold(
    const FieldTrialsView& trials) {
  FieldTrialFlag disable_low_tier("Disabled");
  FieldTrialParameter<int> max_core_count("max_core_count", 2);
  ParseFieldTrial({&disable_low_tier, &max_core_count},
                  trials.Lookup("WebRTC-VP9-LowTierOptimizations"));
  if (disable_low_tier.Get()) {
    return std::nullopt;
  }
  return max_core_count.Get();
}

std::optional<int> ParseEncoderThreadLimit(const FieldTrialsView& trials) {
  FieldTrialOptional<int> encoder_thread_limit("encoder_thread_limit");
  ParseFieldTrial({&encoder_thread_limit},
                  trials.Lookup("WebRTC-VideoEncoderSettings"));
  return encoder_thread_limit.GetOptional();
}

}  //  namespace

VideoStreamEncoder::EncoderRateSettings::EncoderRateSettings()
    : rate_control(),
      encoder_target(DataRate::Zero()),
      stable_encoder_target(DataRate::Zero()) {}

VideoStreamEncoder::EncoderRateSettings::EncoderRateSettings(
    const VideoBitrateAllocation& bitrate,
    double framerate_fps,
    DataRate bandwidth_allocation,
    DataRate encoder_target,
    DataRate stable_encoder_target)
    : rate_control(bitrate, framerate_fps, bandwidth_allocation),
      encoder_target(encoder_target),
      stable_encoder_target(stable_encoder_target) {}

bool VideoStreamEncoder::EncoderRateSettings::operator==(
    const EncoderRateSettings& rhs) const {
  return rate_control == rhs.rate_control &&
         encoder_target == rhs.encoder_target &&
         stable_encoder_target == rhs.stable_encoder_target;
}

bool VideoStreamEncoder::EncoderRateSettings::operator!=(
    const EncoderRateSettings& rhs) const {
  return !(*this == rhs);
}

class VideoStreamEncoder::DegradationPreferenceManager
    : public DegradationPreferenceProvider {
 public:
  explicit DegradationPreferenceManager(
      VideoStreamAdapter* video_stream_adapter)
      : degradation_preference_(DegradationPreference::DISABLED),
        is_screenshare_(false),
        effective_degradation_preference_(DegradationPreference::DISABLED),
        video_stream_adapter_(video_stream_adapter) {
    RTC_DCHECK(video_stream_adapter_);
    sequence_checker_.Detach();
  }

  ~DegradationPreferenceManager() override = default;

  DegradationPreference degradation_preference() const override {
    RTC_DCHECK_RUN_ON(&sequence_checker_);
    return effective_degradation_preference_;
  }

  void SetDegradationPreference(DegradationPreference degradation_preference) {
    RTC_DCHECK_RUN_ON(&sequence_checker_);
    degradation_preference_ = degradation_preference;
    MaybeUpdateEffectiveDegradationPreference();
  }

  void SetIsScreenshare(bool is_screenshare) {
    RTC_DCHECK_RUN_ON(&sequence_checker_);
    is_screenshare_ = is_screenshare;
    MaybeUpdateEffectiveDegradationPreference();
  }

 private:
  void MaybeUpdateEffectiveDegradationPreference()
      RTC_RUN_ON(&sequence_checker_) {
    DegradationPreference effective_degradation_preference =
        (is_screenshare_ &&
         degradation_preference_ == DegradationPreference::BALANCED)
            ? DegradationPreference::MAINTAIN_RESOLUTION
            : degradation_preference_;

    if (effective_degradation_preference != effective_degradation_preference_) {
      effective_degradation_preference_ = effective_degradation_preference;
      video_stream_adapter_->SetDegradationPreference(
          effective_degradation_preference);
    }
  }

  RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_;
  DegradationPreference degradation_preference_
      RTC_GUARDED_BY(&sequence_checker_);
  bool is_screenshare_ RTC_GUARDED_BY(&sequence_checker_);
  DegradationPreference effective_degradation_preference_
      RTC_GUARDED_BY(&sequence_checker_);
  VideoStreamAdapter* video_stream_adapter_ RTC_GUARDED_BY(&sequence_checker_);
};

VideoStreamEncoder::VideoStreamEncoder(
    const Environment& env,
    uint32_t number_of_cores,
    VideoStreamEncoderObserver* encoder_stats_observer,
    const VideoStreamEncoderSettings& settings,
    std::unique_ptr<OveruseFrameDetector> overuse_detector,
    std::unique_ptr<FrameCadenceAdapterInterface> frame_cadence_adapter,
    std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
        encoder_queue,
    BitrateAllocationCallbackType allocation_cb_type,
    webrtc::VideoEncoderFactory::EncoderSelectorInterface* encoder_selector)
    : env_(env),
      worker_queue_(TaskQueueBase::Current()),
      number_of_cores_(number_of_cores),
      settings_(settings),
      allocation_cb_type_(allocation_cb_type),
      rate_control_settings_(env_.field_trials()),
      encoder_selector_from_constructor_(encoder_selector),
      encoder_selector_from_factory_(
          encoder_selector_from_constructor_
              ? nullptr
              : settings.encoder_factory->GetEncoderSelector()),
      encoder_selector_(encoder_selector_from_constructor_
                            ? encoder_selector_from_constructor_
                            : encoder_selector_from_factory_.get()),
      encoder_stats_observer_(encoder_stats_observer),
      frame_cadence_adapter_(std::move(frame_cadence_adapter)),
      delta_ntp_internal_ms_(env_.clock().CurrentNtpInMilliseconds() -
                             env_.clock().TimeInMilliseconds()),
      last_frame_log_ms_(env_.clock().TimeInMilliseconds()),
      next_frame_types_(1, VideoFrameType::kVideoFrameDelta),
      input_state_provider_(encoder_stats_observer),
      video_stream_adapter_(
          std::make_unique<VideoStreamAdapter>(&input_state_provider_,
                                               encoder_stats_observer,
                                               env_.field_trials())),
      degradation_preference_manager_(
          std::make_unique<DegradationPreferenceManager>(
              video_stream_adapter_.get())),
      stream_resource_manager_(&input_state_provider_,
                               encoder_stats_observer,
                               &env_.clock(),
                               settings_.experiment_cpu_load_estimator,
                               std::move(overuse_detector),
                               degradation_preference_manager_.get(),
                               env_.field_trials()),
      video_source_sink_controller_(/*sink=*/frame_cadence_adapter_.get(),
                                    /*source=*/nullptr),
      default_limits_allowed_(!env_.field_trials().IsEnabled(
          "WebRTC-DefaultBitrateLimitsKillSwitch")),
      qp_parsing_allowed_(
          !env_.field_trials().IsEnabled("WebRTC-QpParsingKillSwitch")),
      switch_encoder_on_init_failures_(!env_.field_trials().IsDisabled(
          kSwitchEncoderOnInitializationFailuresFieldTrial)),
      vp9_low_tier_core_threshold_(
          ParseVp9LowTierCoreCountThreshold(env_.field_trials())),
      experimental_encoder_thread_limit_(
          ParseEncoderThreadLimit(env_.field_trials())),
      encoder_queue_(std::move(encoder_queue)) {
  TRACE_EVENT0("webrtc", "VideoStreamEncoder::VideoStreamEncoder");
  RTC_DCHECK_RUN_ON(worker_queue_);
  RTC_DCHECK(encoder_stats_observer);
  RTC_DCHECK_GE(number_of_cores, 1);

  frame_cadence_adapter_->Initialize(&cadence_callback_);
  stream_resource_manager_.Initialize(encoder_queue_.get());

  encoder_queue_->PostTask([this] {
    RTC_DCHECK_RUN_ON(encoder_queue_.get());

    resource_adaptation_processor_ =
        std::make_unique<ResourceAdaptationProcessor>(
            video_stream_adapter_.get());

    stream_resource_manager_.SetAdaptationProcessor(
        resource_adaptation_processor_.get(), video_stream_adapter_.get());
    resource_adaptation_processor_->AddResourceLimitationsListener(
        &stream_resource_manager_);
    video_stream_adapter_->AddRestrictionsListener(&stream_resource_manager_);
    video_stream_adapter_->AddRestrictionsListener(this);
    stream_resource_manager_.MaybeInitializePixelLimitResource();

    // Add the stream resource manager's resources to the processor.
    adaptation_constraints_ = stream_resource_manager_.AdaptationConstraints();
    for (auto* constraint : adaptation_constraints_) {
      video_stream_adapter_->AddAdaptationConstraint(constraint);
    }
  });
}

VideoStreamEncoder::~VideoStreamEncoder() {
  RTC_DCHECK_RUN_ON(worker_queue_);
  RTC_DCHECK(!video_source_sink_controller_.HasSource())
      << "Must call ::Stop() before destruction.";

  // The queue must be destroyed before its pointer is invalidated to avoid race
  // between destructor and running task that check if function is called on the
  // encoder_queue_.
  // std::unique_ptr destructor does the same two operations in reverse order as
  // it doesn't expect member would be used after its destruction has started.
  encoder_queue_.get_deleter()(encoder_queue_.get());
  encoder_queue_.release();
}

void VideoStreamEncoder::Stop() {
  RTC_DCHECK_RUN_ON(worker_queue_);
  video_source_sink_controller_.SetSource(nullptr);

  rtc::Event shutdown_event;
  absl::Cleanup shutdown = [&shutdown_event] { shutdown_event.Set(); };
  encoder_queue_->PostTask([this, shutdown = std::move(shutdown)] {
    RTC_DCHECK_RUN_ON(encoder_queue_.get());
    if (resource_adaptation_processor_) {
      // We're no longer interested in restriction updates, which may get
      // triggered as part of removing resources.
      video_stream_adapter_->RemoveRestrictionsListener(this);
      video_stream_adapter_->RemoveRestrictionsListener(
          &stream_resource_manager_);
      resource_adaptation_processor_->RemoveResourceLimitationsListener(
          &stream_resource_manager_);
      // Stop and remove resources and delete adaptation processor.
      stream_resource_manager_.StopManagedResources();
      for (auto* constraint : adaptation_constraints_) {
        video_stream_adapter_->RemoveAdaptationConstraint(constraint);
      }
      for (auto& resource : additional_resources_) {
        stream_resource_manager_.RemoveResource(resource);
      }
      additional_resources_.clear();
      stream_resource_manager_.SetAdaptationProcessor(nullptr, nullptr);
      resource_adaptation_processor_.reset();
    }
    rate_allocator_ = nullptr;
    ReleaseEncoder();
    encoder_ = nullptr;
    frame_cadence_adapter_ = nullptr;
    frame_instrumentation_generator_ = nullptr;
  });
  shutdown_event.Wait(rtc::Event::kForever);
}

void VideoStreamEncoder::SetFecControllerOverride(
    FecControllerOverride* fec_controller_override) {
  encoder_queue_->PostTask([this, fec_controller_override] {
    RTC_DCHECK_RUN_ON(encoder_queue_.get());
    RTC_DCHECK(!fec_controller_override_);
    fec_controller_override_ = fec_controller_override;
    if (encoder_) {
      encoder_->SetFecControllerOverride(fec_controller_override_);
    }
  });
}

void VideoStreamEncoder::AddAdaptationResource(
    rtc::scoped_refptr<Resource> resource) {
  RTC_DCHECK_RUN_ON(worker_queue_);
  TRACE_EVENT0("webrtc", "VideoStreamEncoder::AddAdaptationResource");
  // Map any externally added resources as kCpu for the sake of stats reporting.
  // TODO(hbos): Make the manager map any unknown resources to kCpu and get rid
  // of this MapResourceToReason() call.
  TRACE_EVENT_ASYNC_BEGIN0(
      "webrtc", "VideoStreamEncoder::AddAdaptationResource(latency)", this);
  encoder_queue_->PostTask([this, resource = std::move(resource)] {
    TRACE_EVENT_ASYNC_END0(
        "webrtc", "VideoStreamEncoder::AddAdaptationResource(latency)", this);
    RTC_DCHECK_RUN_ON(encoder_queue_.get());
    additional_resources_.push_back(resource);
    stream_resource_manager_.AddResource(resource, VideoAdaptationReason::kCpu);
  });
}

std::vector<rtc::scoped_refptr<Resource>>
VideoStreamEncoder::GetAdaptationResources() {
  RTC_DCHECK_RUN_ON(worker_queue_);
  // In practice, this method is only called by tests to verify operations that
  // run on the encoder queue. So rather than force PostTask() operations to
  // be accompanied by an event and a `Wait()`, we'll use PostTask + Wait()
  // here.
  rtc::Event event;
  std::vector<rtc::scoped_refptr<Resource>> resources;
  encoder_queue_->PostTask([&] {
    RTC_DCHECK_RUN_ON(encoder_queue_.get());
    resources = resource_adaptation_processor_->GetResources();
    event.Set();
  });
  event.Wait(rtc::Event::kForever);
  return resources;
}

void VideoStreamEncoder::SetSource(
    rtc::VideoSourceInterface<VideoFrame>* source,
    const DegradationPreference& degradation_preference) {
  RTC_DCHECK_RUN_ON(worker_queue_);
  video_source_sink_controller_.SetSource(source);
  input_state_provider_.OnHasInputChanged(source);

  // This may trigger reconfiguring the QualityScaler on the encoder queue.
  encoder_queue_->PostTask([this, degradation_preference] {
    RTC_DCHECK_RUN_ON(encoder_queue_.get());
    degradation_preference_manager_->SetDegradationPreference(
        degradation_preference);
    stream_resource_manager_.SetDegradationPreferences(degradation_preference);
    if (encoder_) {
      stream_resource_manager_.ConfigureQualityScaler(
          encoder_->GetEncoderInfo());
      stream_resource_manager_.ConfigureBandwidthQualityScaler(
          encoder_->GetEncoderInfo());
    }
  });
}

void VideoStreamEncoder::SetSink(EncoderSink* sink, bool rotation_applied) {
  RTC_DCHECK_RUN_ON(worker_queue_);
  video_source_sink_controller_.SetRotationApplied(rotation_applied);
  video_source_sink_controller_.PushSourceSinkSettings();

  encoder_queue_->PostTask([this, sink] {
    RTC_DCHECK_RUN_ON(encoder_queue_.get());
    sink_ = sink;
  });
}

void VideoStreamEncoder::SetStartBitrate(int start_bitrate_bps) {
  encoder_queue_->PostTask([this, start_bitrate_bps] {
    RTC_DCHECK_RUN_ON(encoder_queue_.get());
    RTC_LOG(LS_INFO) << "SetStartBitrate " << start_bitrate_bps;
    encoder_target_bitrate_bps_ =
        start_bitrate_bps != 0 ? std::optional<uint32_t>(start_bitrate_bps)
                               : std::nullopt;
    stream_resource_manager_.SetStartBitrate(
        DataRate::BitsPerSec(start_bitrate_bps));
  });
}

void VideoStreamEncoder::ConfigureEncoder(VideoEncoderConfig config,
                                          size_t max_data_payload_length) {
  ConfigureEncoder(std::move(config), max_data_payload_length, nullptr);
}

void VideoStreamEncoder::ConfigureEncoder(VideoEncoderConfig config,
                                          size_t max_data_payload_length,
                                          SetParametersCallback callback) {
  RTC_DCHECK_RUN_ON(worker_queue_);

  // Inform source about max configured framerate,
  // scale_resolution_down_to and which layers are active.
  int max_framerate = -1;
  // Is any layer active.
  bool active = false;
  // The max scale_resolution_down_to.
  std::optional<rtc::VideoSinkWants::FrameSize> scale_resolution_down_to;
  for (const auto& stream : config.simulcast_layers) {
    active |= stream.active;
    if (stream.active) {
      max_framerate = std::max(stream.max_framerate, max_framerate);
    }
    // Note: we propagate the highest scale_resolution_down_to regardless
    // if layer is active or not.
    if (stream.scale_resolution_down_to) {
      if (!scale_resolution_down_to) {
        scale_resolution_down_to.emplace(
            stream.scale_resolution_down_to->width,
            stream.scale_resolution_down_to->height);
      } else {
        scale_resolution_down_to.emplace(
            std::max(stream.scale_resolution_down_to->width,
                     scale_resolution_down_to->width),
            std::max(stream.scale_resolution_down_to->height,
                     scale_resolution_down_to->height));
      }
    }
  }
  if (scale_resolution_down_to !=
          video_source_sink_controller_.scale_resolution_down_to() ||
      active != video_source_sink_controller_.active() ||
      max_framerate !=
          video_source_sink_controller_.frame_rate_upper_limit().value_or(-1)) {
    video_source_sink_controller_.SetScaleResolutionDownTo(
        scale_resolution_down_to);
    if (max_framerate >= 0) {
      video_source_sink_controller_.SetFrameRateUpperLimit(max_framerate);
    } else {
      video_source_sink_controller_.SetFrameRateUpperLimit(std::nullopt);
    }
    video_source_sink_controller_.SetActive(active);
    video_source_sink_controller_.PushSourceSinkSettings();
  }

  encoder_queue_->PostTask([this, config = std::move(config),
                            max_data_payload_length,
                            callback = std::move(callback)]() mutable {
    RTC_DCHECK_RUN_ON(encoder_queue_.get());
    RTC_DCHECK(sink_);
    RTC_LOG(LS_INFO) << "ConfigureEncoder requested.";

    // Set up the frame cadence adapter according to if we're going to do
    // screencast. The final number of spatial layers is based on info
    // in `send_codec_`, which is computed based on incoming frame
    // dimensions which can only be determined later.
    //
    // Note: zero-hertz mode isn't enabled by this alone. Constraints also
    // have to be set up with min_fps = 0 and max_fps > 0.
    if (config.content_type == VideoEncoderConfig::ContentType::kScreen) {
      frame_cadence_adapter_->SetZeroHertzModeEnabled(
          FrameCadenceAdapterInterface::ZeroHertzModeParams{});
    } else {
      frame_cadence_adapter_->SetZeroHertzModeEnabled(std::nullopt);
    }

    pending_encoder_creation_ =
        (!encoder_ || encoder_config_.video_format != config.video_format ||
         max_data_payload_length_ != max_data_payload_length);
    encoder_config_ = std::move(config);
    max_data_payload_length_ = max_data_payload_length;
    pending_encoder_reconfiguration_ = true;

    if (settings_.enable_frame_instrumentation_generator) {
      frame_instrumentation_generator_ =
          std::make_unique<FrameInstrumentationGenerator>(
              encoder_config_.codec_type);
    }

    // Reconfigure the encoder now if the frame resolution is known.
    // Otherwise, the reconfiguration is deferred until the next frame to
    // minimize the number of reconfigurations. The codec configuration
    // depends on incoming video frame size.
    if (last_frame_info_) {
      if (callback) {
        encoder_configuration_callbacks_.push_back(std::move(callback));
      }

      ReconfigureEncoder();
    } else {
      webrtc::InvokeSetParametersCallback(callback, webrtc::RTCError::OK());
    }
  });
}

// We should reduce the number of 'full' ReconfigureEncoder(). If only need
// subset of it at runtime, consider handle it in
// VideoStreamEncoder::EncodeVideoFrame() when encoder_info_ != info.
void VideoStreamEncoder::ReconfigureEncoder() {
  // Running on the encoder queue.
  RTC_DCHECK(pending_encoder_reconfiguration_);
  RTC_LOG(LS_INFO) << "[VSE] " << __func__
                   << " [encoder_config=" << encoder_config_.ToString() << "]";

  bool encoder_reset_required = false;
  if (pending_encoder_creation_) {
    // Destroy existing encoder instance before creating a new one. Otherwise
    // attempt to create another instance will fail if encoder factory
    // supports only single instance of encoder of given type.
    encoder_.reset();

    encoder_ = MaybeCreateFrameDumpingEncoderWrapper(
        settings_.encoder_factory->Create(env_, encoder_config_.video_format),
        env_.field_trials());
    if (!encoder_) {
      RTC_LOG(LS_ERROR) << "CreateVideoEncoder failed, failing encoder format: "
                        << encoder_config_.video_format.ToString();
      RequestEncoderSwitch();
      return;
    }

    if (encoder_selector_) {
      encoder_selector_->OnCurrentEncoder(encoder_config_.video_format);
    }

    encoder_->SetFecControllerOverride(fec_controller_override_);

    encoder_reset_required = true;
  }

  // TODO(webrtc:14451) : Move AlignmentAdjuster into EncoderStreamFactory
  // Possibly adjusts scale_resolution_down_by in `encoder_config_` to limit the
  // alignment value.
  AlignmentAdjuster::GetAlignmentAndMaybeAdjustScaleFactors(
      encoder_->GetEncoderInfo(), &encoder_config_, std::nullopt);

  std::vector<VideoStream> streams;
  if (encoder_config_.video_stream_factory) {
    // Note: only tests set their own EncoderStreamFactory...
    encoder_config_.video_stream_factory->SetEncoderInfo(encoder_->GetEncoderInfo());
    streams = encoder_config_.video_stream_factory->CreateEncoderStreams(
        env_.field_trials(), last_frame_info_->width, last_frame_info_->height,
        encoder_config_);
  } else {
    auto factory = rtc::make_ref_counted<cricket::EncoderStreamFactory>(
        encoder_->GetEncoderInfo(), latest_restrictions_);

    streams = factory->CreateEncoderStreams(
        env_.field_trials(), last_frame_info_->width, last_frame_info_->height,
        encoder_config_);
  }

  // TODO(webrtc:14451) : Move AlignmentAdjuster into EncoderStreamFactory
  // Get alignment when actual number of layers are known.
  int alignment = AlignmentAdjuster::GetAlignmentAndMaybeAdjustScaleFactors(
      encoder_->GetEncoderInfo(), &encoder_config_, streams.size());

  // Check that the higher layers do not try to set number of temporal layers
  // to less than 1.
  // TODO(brandtr): Get rid of the wrapping optional as it serves no purpose
  // at this layer.
#if RTC_DCHECK_IS_ON
  for (const auto& stream : streams) {
    RTC_DCHECK_GE(stream.num_temporal_layers.value_or(1), 1);
  }
#endif

  // TODO(ilnik): If configured resolution is significantly less than provided,
  // e.g. because there are not enough SSRCs for all simulcast streams,
  // signal new resolutions via SinkWants to video source.

  // Stream dimensions may be not equal to given because of a simulcast
  // restrictions.
  auto highest_stream = absl::c_max_element(
      streams, [](const webrtc::VideoStream& a, const webrtc::VideoStream& b) {
        return std::tie(a.width, a.height) < std::tie(b.width, b.height);
      });
  int highest_stream_width = static_cast<int>(highest_stream->width);
  int highest_stream_height = static_cast<int>(highest_stream->height);
  // Dimension may be reduced to be, e.g. divisible by 4.
  RTC_CHECK_GE(last_frame_info_->width, highest_stream_width);
  RTC_CHECK_GE(last_frame_info_->height, highest_stream_height);
  crop_width_ = last_frame_info_->width - highest_stream_width;
  crop_height_ = last_frame_info_->height - highest_stream_height;

  if (!encoder_->GetEncoderInfo().is_qp_trusted.value_or(true)) {
    // when qp is not trusted, we priorities to using the
    // |resolution_bitrate_limits| provided by the decoder.
    const std::vector<VideoEncoder::ResolutionBitrateLimits>& bitrate_limits =
        encoder_->GetEncoderInfo().resolution_bitrate_limits.empty()
            ? EncoderInfoSettings::
                  GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted()
            : encoder_->GetEncoderInfo().resolution_bitrate_limits;

    // For BandwidthQualityScaler, its implement based on a certain pixel_count
    // correspond a certain bps interval. In fact, WebRTC default max_bps is
    // 2500Kbps when width * height > 960 * 540. For example, we assume:
    // 1.the camera support 1080p.
    // 2.ResolutionBitrateLimits set 720p bps interval is [1500Kbps,2000Kbps].
    // 3.ResolutionBitrateLimits set 1080p bps interval is [2000Kbps,2500Kbps].
    // We will never be stable at 720p due to actual encoding bps of 720p and
    // 1080p are both 2500Kbps. So it is necessary to do a linear interpolation
    // to get a certain bitrate for certain pixel_count. It also doesn't work
    // for 960*540 and 640*520, we will nerver be stable at 640*520 due to their
    // |target_bitrate_bps| are both 2000Kbps.
    std::optional<VideoEncoder::ResolutionBitrateLimits>
        qp_untrusted_bitrate_limit = EncoderInfoSettings::
            GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
                last_frame_info_->width * last_frame_info_->height,
                bitrate_limits);

    if (qp_untrusted_bitrate_limit) {
      // bandwidth_quality_scaler is only used for singlecast.
      if (streams.size() == 1 && encoder_config_.simulcast_layers.size() == 1) {
        VideoStream& stream = streams.back();
        stream.max_bitrate_bps =
            std::min(stream.max_bitrate_bps,
                     qp_untrusted_bitrate_limit->max_bitrate_bps);
        stream.min_bitrate_bps =
            std::min(stream.max_bitrate_bps,
                     qp_untrusted_bitrate_limit->min_bitrate_bps);
        // If it is screen share mode, the minimum value of max_bitrate should
        // be greater than/equal to 1200kbps.
        if (encoder_config_.content_type ==
            VideoEncoderConfig::ContentType::kScreen) {
          stream.max_bitrate_bps =
              std::max(stream.max_bitrate_bps, kDefaultMinScreenSharebps);
        }
        stream.target_bitrate_bps = stream.max_bitrate_bps;
      }
    }
  } else {
    std::optional<VideoEncoder::ResolutionBitrateLimits>
        encoder_bitrate_limits =
            encoder_->GetEncoderInfo().GetEncoderBitrateLimitsForResolution(
                last_frame_info_->width * last_frame_info_->height);

    if (encoder_bitrate_limits) {
      if (streams.size() == 1 && encoder_config_.simulcast_layers.size() == 1) {
        // Bitrate limits can be set by app (in SDP or RtpEncodingParameters)
        // or/and can be provided by encoder. In presence of both set of
        // limits, the final set is derived as their intersection.
        int min_bitrate_bps;
        if (encoder_config_.simulcast_layers[0].min_bitrate_bps <= 0) {
          min_bitrate_bps = encoder_bitrate_limits->min_bitrate_bps;
        } else {
          min_bitrate_bps = std::max(encoder_bitrate_limits->min_bitrate_bps,
                                     streams.back().min_bitrate_bps);
        }

        int max_bitrate_bps;
        // The API max bitrate comes from both `encoder_config_.max_bitrate_bps`
        // and `encoder_config_.simulcast_layers[0].max_bitrate_bps`.
        std::optional<int> api_max_bitrate_bps;
        if (encoder_config_.simulcast_layers[0].max_bitrate_bps > 0) {
          api_max_bitrate_bps =
              encoder_config_.simulcast_layers[0].max_bitrate_bps;
        }
        if (encoder_config_.max_bitrate_bps > 0) {
          api_max_bitrate_bps = api_max_bitrate_bps.has_value()
                                    ? std::min(encoder_config_.max_bitrate_bps,
                                               *api_max_bitrate_bps)
                                    : encoder_config_.max_bitrate_bps;
        }
        if (!api_max_bitrate_bps.has_value()) {
          max_bitrate_bps = encoder_bitrate_limits->max_bitrate_bps;
        } else {
          max_bitrate_bps = std::min(encoder_bitrate_limits->max_bitrate_bps,
                                     streams.back().max_bitrate_bps);
        }

        if (min_bitrate_bps < max_bitrate_bps) {
          streams.back().min_bitrate_bps = min_bitrate_bps;
          streams.back().max_bitrate_bps = max_bitrate_bps;
          streams.back().target_bitrate_bps =
              std::min(streams.back().target_bitrate_bps,
                       encoder_bitrate_limits->max_bitrate_bps);
        } else {
          RTC_LOG(LS_WARNING)
              << "Bitrate limits provided by encoder"
              << " (min=" << encoder_bitrate_limits->min_bitrate_bps
              << ", max=" << encoder_bitrate_limits->max_bitrate_bps
              << ") do not intersect with limits set by app"
              << " (min=" << streams.back().min_bitrate_bps
              << ", max=" << api_max_bitrate_bps.value_or(-1)
              << "). The app bitrate limits will be used.";
        }
      }
    }
  }

  ApplyEncoderBitrateLimitsIfSingleActiveStream(
      GetEncoderInfoWithBitrateLimitUpdate(
          encoder_->GetEncoderInfo(), encoder_config_, default_limits_allowed_),
      encoder_config_.simulcast_layers, &streams);

  VideoCodec codec = VideoCodecInitializer::SetupCodec(
      env_.field_trials(), encoder_config_, streams);

  if (encoder_config_.codec_type == kVideoCodecVP9 ||
      encoder_config_.codec_type == kVideoCodecAV1
#ifdef RTC_ENABLE_H265
      || encoder_config_.codec_type == kVideoCodecH265
#endif
  ) {
    // Spatial layers configuration might impose some parity restrictions,
    // thus some cropping might be needed.
    RTC_CHECK_GE(last_frame_info_->width, codec.width);
    RTC_CHECK_GE(last_frame_info_->height, codec.height);
    crop_width_ = last_frame_info_->width - codec.width;
    crop_height_ = last_frame_info_->height - codec.height;
    ApplySpatialLayerBitrateLimits(
        GetEncoderInfoWithBitrateLimitUpdate(encoder_->GetEncoderInfo(),
                                             encoder_config_,
                                             default_limits_allowed_),
        encoder_config_, &codec);
  }

  char log_stream_buf[4 * 1024];
  rtc::SimpleStringBuilder log_stream(log_stream_buf);
  log_stream << "ReconfigureEncoder: simulcast streams: ";
  for (size_t i = 0; i < codec.numberOfSimulcastStreams; ++i) {
    std::optional<ScalabilityMode> scalability_mode =
        codec.simulcastStream[i].GetScalabilityMode();
    if (scalability_mode) {
      log_stream << "{" << i << ": " << codec.simulcastStream[i].width << "x"
                 << codec.simulcastStream[i].height << " "
                 << ScalabilityModeToString(*scalability_mode)
                 << ", min_kbps: " << codec.simulcastStream[i].minBitrate
                 << ", target_kbps: " << codec.simulcastStream[i].targetBitrate
                 << ", max_kbps: " << codec.simulcastStream[i].maxBitrate
                 << ", max_fps: " << codec.simulcastStream[i].maxFramerate
                 << ", max_qp: " << codec.simulcastStream[i].qpMax
                 << ", num_tl: "
                 << codec.simulcastStream[i].numberOfTemporalLayers
                 << ", active: "
                 << (codec.simulcastStream[i].active ? "true" : "false") << "}";
    }
  }
  if (encoder_config_.codec_type == kVideoCodecVP9 ||
      encoder_config_.codec_type == kVideoCodecAV1
#ifdef RTC_ENABLE_H265
      || encoder_config_.codec_type == kVideoCodecH265
#endif
  ) {
    log_stream << ", spatial layers: ";
    for (int i = 0; i < GetNumSpatialLayers(codec); ++i) {
      log_stream << "{" << i << ": " << codec.spatialLayers[i].width << "x"
                 << codec.spatialLayers[i].height
                 << ", min_kbps: " << codec.spatialLayers[i].minBitrate
                 << ", target_kbps: " << codec.spatialLayers[i].targetBitrate
                 << ", max_kbps: " << codec.spatialLayers[i].maxBitrate
                 << ", max_fps: " << codec.spatialLayers[i].maxFramerate
                 << ", max_qp: " << codec.spatialLayers[i].qpMax << ", num_tl: "
                 << codec.spatialLayers[i].numberOfTemporalLayers
                 << ", active: "
                 << (codec.spatialLayers[i].active ? "true" : "false") << "}";
    }
  }
  RTC_LOG(LS_INFO) << "[VSE] " << log_stream.str();

  codec.startBitrate = std::max(encoder_target_bitrate_bps_.value_or(0) / 1000,
                                codec.minBitrate);
  codec.startBitrate = std::min(codec.startBitrate, codec.maxBitrate);
  codec.expect_encode_from_texture = last_frame_info_->is_texture;
  // Make sure the start bit rate is sane...
  RTC_DCHECK_LE(codec.startBitrate, 1000000);
  max_framerate_ = codec.maxFramerate;

  // The resolutions that we're actually encoding with.
  std::vector<rtc::VideoSinkWants::FrameSize> encoder_resolutions;
  // TODO(hbos): For the case of SVC, also make use of `codec.spatialLayers`.
  // For now, SVC layers are handled by the VP9 encoder.
  for (const auto& simulcastStream : codec.simulcastStream) {
    if (!simulcastStream.active)
      continue;
    encoder_resolutions.emplace_back(simulcastStream.width,
                                     simulcastStream.height);
  }

  worker_queue_->PostTask(SafeTask(
      task_safety_.flag(),
      [this, alignment,
       encoder_resolutions = std::move(encoder_resolutions)]() {
        RTC_DCHECK_RUN_ON(worker_queue_);
        if (alignment != video_source_sink_controller_.resolution_alignment() ||
            encoder_resolutions !=
                video_source_sink_controller_.resolutions()) {
          video_source_sink_controller_.SetResolutionAlignment(alignment);
          video_source_sink_controller_.SetResolutions(
              std::move(encoder_resolutions));
          video_source_sink_controller_.PushSourceSinkSettings();
        }
      }));

  rate_allocator_ = settings_.bitrate_allocator_factory->Create(env_, codec);
  rate_allocator_->SetLegacyConferenceMode(
      encoder_config_.legacy_conference_mode);

  // Reset (release existing encoder) if one exists and anything except
  // start bitrate or max framerate has changed.
  if (!encoder_reset_required) {
    encoder_reset_required = RequiresEncoderReset(
        send_codec_, codec, was_encode_called_since_last_initialization_);
  }

  if (codec.codecType == VideoCodecType::kVideoCodecVP9 &&
      number_of_cores_ <= vp9_low_tier_core_threshold_.value_or(0)) {
    codec.SetVideoEncoderComplexity(VideoCodecComplexity::kComplexityLow);
  }

  quality_convergence_controller_.Initialize(
      codec.numberOfSimulcastStreams, encoder_->GetEncoderInfo().min_qp,
      codec.codecType, env_.field_trials());

  send_codec_ = codec;

  // Keep the same encoder, as long as the video_format is unchanged.
  // Encoder creation block is split in two since EncoderInfo needed to start
  // CPU adaptation with the correct settings should be polled after
  // encoder_->InitEncode().
  if (encoder_reset_required) {
    ReleaseEncoder();
    const size_t max_data_payload_length = max_data_payload_length_ > 0
                                               ? max_data_payload_length_
                                               : kDefaultPayloadSize;
    VideoEncoder::Settings settings = VideoEncoder::Settings(
        settings_.capabilities, number_of_cores_, max_data_payload_length);
    settings.encoder_thread_limit = experimental_encoder_thread_limit_;
    int error = encoder_->InitEncode(&send_codec_, settings);
    if (error != 0) {
      RTC_LOG(LS_ERROR) << "Failed to initialize the encoder associated with "
                           "codec type: "
                        << CodecTypeToPayloadString(send_codec_.codecType)
                        << " (" << send_codec_.codecType
                        << "). Error: " << error;
      ReleaseEncoder();
    } else {
      encoder_initialized_ = true;
      encoder_->RegisterEncodeCompleteCallback(this);
      frame_encode_metadata_writer_.OnEncoderInit(send_codec_);
      next_frame_types_.clear();
      next_frame_types_.resize(
          std::max(static_cast<int>(codec.numberOfSimulcastStreams), 1),
          VideoFrameType::kVideoFrameKey);
    }

    frame_encode_metadata_writer_.Reset();
    last_encode_info_ms_ = std::nullopt;
    was_encode_called_since_last_initialization_ = false;
  }

  // Inform dependents of updated encoder settings.
  OnEncoderSettingsChanged();

  if (encoder_initialized_) {
    RTC_LOG(LS_VERBOSE) << " max bitrate " << codec.maxBitrate
                        << " start bitrate " << codec.startBitrate
                        << " max frame rate " << codec.maxFramerate
                        << " max payload size " << max_data_payload_length_;
  } else {
    RTC_LOG(LS_ERROR) << "[VSE] Failed to configure encoder.";
    rate_allocator_ = nullptr;
  }

  if (pending_encoder_creation_) {
    stream_resource_manager_.ConfigureEncodeUsageResource();
    pending_encoder_creation_ = false;
  }

  int num_layers;
  if (codec.codecType == kVideoCodecVP8) {
    num_layers = codec.VP8()->numberOfTemporalLayers;
  } else if (codec.codecType == kVideoCodecVP9) {
    num_layers = codec.VP9()->numberOfTemporalLayers;
  } else if ((codec.codecType == kVideoCodecAV1 ||
              codec.codecType == kVideoCodecH265) &&
             codec.GetScalabilityMode().has_value()) {
    num_layers =
        ScalabilityModeToNumTemporalLayers(*(codec.GetScalabilityMode()));
  } else if (codec.codecType == kVideoCodecH264) {
    num_layers = codec.H264()->numberOfTemporalLayers;
  } else if (codec.codecType == kVideoCodecGeneric &&
             codec.numberOfSimulcastStreams > 0) {
    // This is mainly for unit testing, disabling frame dropping.
    // TODO(sprang): Add a better way to disable frame dropping.
    num_layers = codec.simulcastStream[0].numberOfTemporalLayers;
  } else {
    num_layers = 1;
  }

  frame_dropper_.Reset();
  frame_dropper_.SetRates(codec.startBitrate, max_framerate_);
  // Force-disable frame dropper if either:
  //  * We have screensharing with layers.
  //  * "WebRTC-FrameDropper" field trial is "Disabled".
  force_disable_frame_dropper_ =
      env_.field_trials().IsDisabled(kFrameDropperFieldTrial) ||
      (num_layers > 1 && codec.mode == VideoCodecMode::kScreensharing);

  const VideoEncoder::EncoderInfo info = encoder_->GetEncoderInfo();
  if (rate_control_settings_.UseEncoderBitrateAdjuster()) {
    bitrate_adjuster_ = std::make_unique<EncoderBitrateAdjuster>(
        codec, env_.field_trials(), env_.clock());
    bitrate_adjuster_->OnEncoderInfo(info);
  }

  if (rate_allocator_ && last_encoder_rate_settings_) {
    // We have a new rate allocator instance and already configured target
    // bitrate. Update the rate allocation and notify observers.
    // We must invalidate the last_encoder_rate_settings_ to ensure
    // the changes get propagated to all listeners.
    EncoderRateSettings rate_settings = *last_encoder_rate_settings_;
    last_encoder_rate_settings_.reset();
    rate_settings.rate_control.framerate_fps = GetInputFramerateFps();

    SetEncoderRates(UpdateBitrateAllocation(rate_settings));
  }

  encoder_stats_observer_->OnEncoderReconfigured(encoder_config_, streams);

  pending_encoder_reconfiguration_ = false;

  bool is_svc = false;
  bool single_stream_or_non_first_inactive = true;
  for (size_t i = 1; i < encoder_config_.simulcast_layers.size(); ++i) {
    if (encoder_config_.simulcast_layers[i].active) {
      single_stream_or_non_first_inactive = false;
      break;
    }
  }
  // Set min_bitrate_bps, max_bitrate_bps, and max padding bit rate for VP9,
  // AV1 and H.265, and leave only one stream containing all necessary
  // information.
  if ((
#ifdef RTC_ENABLE_H265
          encoder_config_.codec_type == kVideoCodecH265 ||
#endif
          encoder_config_.codec_type == kVideoCodecVP9 ||
          encoder_config_.codec_type == kVideoCodecAV1) &&
      single_stream_or_non_first_inactive) {
    // Lower max bitrate to the level codec actually can produce.
    streams[0].max_bitrate_bps =
        std::min(streams[0].max_bitrate_bps,
                 SvcRateAllocator::GetMaxBitrate(codec).bps<int>());
    streams[0].min_bitrate_bps = codec.spatialLayers[0].minBitrate * 1000;
    // target_bitrate_bps specifies the maximum padding bitrate.
    streams[0].target_bitrate_bps =
        SvcRateAllocator::GetPaddingBitrate(codec).bps<int>();
    streams[0].width = streams.back().width;
    streams[0].height = streams.back().height;
    is_svc = GetNumSpatialLayers(codec) > 1;
    streams.resize(1);
  }

  sink_->OnEncoderConfigurationChanged(
      std::move(streams), is_svc, encoder_config_.content_type,
      encoder_config_.min_transmit_bitrate_bps);

  stream_resource_manager_.ConfigureQualityScaler(info);
  stream_resource_manager_.ConfigureBandwidthQualityScaler(info);

  webrtc::RTCError encoder_configuration_result = webrtc::RTCError::OK();

  if (!encoder_initialized_) {
    RTC_LOG(LS_WARNING) << "Failed to initialize "
                        << CodecTypeToPayloadString(codec.codecType)
                        << " encoder."
                        << "switch_encoder_on_init_failures: "
                        << switch_encoder_on_init_failures_;

    if (switch_encoder_on_init_failures_) {
      RequestEncoderSwitch();
    } else {
      encoder_configuration_result =
          webrtc::RTCError(RTCErrorType::UNSUPPORTED_OPERATION);
    }
  }

  if (!encoder_configuration_callbacks_.empty()) {
    for (auto& callback : encoder_configuration_callbacks_) {
      webrtc::InvokeSetParametersCallback(callback,
                                          encoder_configuration_result);
    }
    encoder_configuration_callbacks_.clear();
  }
}

void VideoStreamEncoder::RequestEncoderSwitch() {
  bool is_encoder_switching_supported =
      settings_.encoder_switch_request_callback != nullptr;
  bool is_encoder_selector_available = encoder_selector_ != nullptr;

  RTC_LOG(LS_INFO) << "RequestEncoderSwitch."
                   << " is_encoder_selector_available: "
                   << is_encoder_selector_available
                   << " is_encoder_switching_supported: "
                   << is_encoder_switching_supported;

  if (!is_encoder_switching_supported) {
    return;
  }

  // If encoder selector is available, switch to the encoder it prefers.
  // Otherwise try switching to VP8 (default WebRTC codec).
  std::optional<SdpVideoFormat> preferred_fallback_encoder;
  if (is_encoder_selector_available) {
    preferred_fallback_encoder = encoder_selector_->OnEncoderBroken();
  }

  if (!preferred_fallback_encoder) {
    preferred_fallback_encoder =
        SdpVideoFormat(CodecTypeToPayloadString(kVideoCodecVP8));
  }

  settings_.encoder_switch_request_callback->RequestEncoderSwitch(
      *preferred_fallback_encoder, /*allow_default_fallback=*/true);
}

void VideoStreamEncoder::OnEncoderSettingsChanged() {
  EncoderSettings encoder_settings(
      GetEncoderInfoWithBitrateLimitUpdate(
          encoder_->GetEncoderInfo(), encoder_config_, default_limits_allowed_),
      encoder_config_.Copy(), send_codec_);
  stream_resource_manager_.SetEncoderSettings(encoder_settings);
  input_state_provider_.OnEncoderSettingsChanged(encoder_settings);
  bool is_screenshare = encoder_settings.encoder_config().content_type ==
                        VideoEncoderConfig::ContentType::kScreen;
  degradation_preference_manager_->SetIsScreenshare(is_screenshare);
  if (is_screenshare) {
    frame_cadence_adapter_->SetZeroHertzModeEnabled(
        FrameCadenceAdapterInterface::ZeroHertzModeParams{
            send_codec_.numberOfSimulcastStreams});
  }
}

void VideoStreamEncoder::OnFrame(Timestamp post_time,
                                 bool queue_overload,
                                 const VideoFrame& video_frame) {
  RTC_DCHECK_RUN_ON(encoder_queue_.get());
  VideoFrame incoming_frame = video_frame;

  // In some cases, e.g., when the frame from decoder is fed to encoder,
  // the timestamp may be set to the future. As the encoding pipeline assumes
  // capture time to be less than present time, we should reset the capture
  // timestamps here. Otherwise there may be issues with RTP send stream.
  if (incoming_frame.timestamp_us() > post_time.us())
    incoming_frame.set_timestamp_us(post_time.us());

  // Capture time may come from clock with an offset and drift from clock_.
  int64_t capture_ntp_time_ms;
  if (video_frame.ntp_time_ms() > 0) {
    capture_ntp_time_ms = video_frame.ntp_time_ms();
  } else if (video_frame.render_time_ms() != 0) {
    capture_ntp_time_ms = video_frame.render_time_ms() + delta_ntp_internal_ms_;
  } else {
    capture_ntp_time_ms = post_time.ms() + delta_ntp_internal_ms_;
  }
  incoming_frame.set_ntp_time_ms(capture_ntp_time_ms);

  // Convert NTP time, in ms, to RTP timestamp.
  const int kMsToRtpTimestamp = 90;
  incoming_frame.set_rtp_timestamp(
      kMsToRtpTimestamp * static_cast<uint32_t>(incoming_frame.ntp_time_ms()));

  // Identifier should remain the same for newly produced incoming frame and the
  // received |video_frame|.
  incoming_frame.set_presentation_timestamp(
      video_frame.presentation_timestamp());

  if (incoming_frame.ntp_time_ms() <= last_captured_timestamp_) {
    // We don't allow the same capture time for two frames, drop this one.
    RTC_LOG(LS_WARNING) << "Same/old NTP timestamp ("
                        << incoming_frame.ntp_time_ms()
                        << " <= " << last_captured_timestamp_
                        << ") for incoming frame. Dropping.";
    ProcessDroppedFrame(incoming_frame,
                        VideoStreamEncoderObserver::DropReason::kBadTimestamp);
    return;
  }

  bool log_stats = false;
  if (post_time.ms() - last_frame_log_ms_ > kFrameLogIntervalMs) {
--> --------------------

--> maximum size reached

--> --------------------