content/renderer/media/webrtc/media_stream_video_webrtc_sink.cc

// Copyright 2014 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "content/renderer/media/webrtc/media_stream_video_webrtc_sink.h"

#include <algorithm>
#include <memory>
#include <string>

#include "base/feature_list.h"
#include "base/location.h"
#include "base/numerics/safe_conversions.h"
#include "base/single_thread_task_runner.h"
#include "base/strings/utf_string_conversions.h"
#include "base/synchronization/lock.h"
#include "base/timer/timer.h"
#include "content/public/common/content_features.h"
#include "content/public/renderer/media_stream_utils.h"
#include "content/renderer/media/stream/media_stream_constraints_util.h"
#include "content/renderer/media/stream/media_stream_video_track.h"
#include "content/renderer/media/webrtc/peer_connection_dependency_factory.h"
#include "media/base/limits.h"
#include "third_party/webrtc/api/videosourceproxy.h"
#include "third_party/webrtc/pc/videotracksource.h"

namespace content {

namespace {

absl::optional<bool> ToAbslOptionalBool(const base::Optional<bool>& value) {
  return value ? absl::optional<bool>(*value) : absl::nullopt;
}

}  // namespace

class MediaStreamVideoWebRtcSink::WebRtcVideoSource
    : public webrtc::VideoTrackSource {
 public:
  WebRtcVideoSource(WebRtcVideoCapturerAdapter* capture_adapter,
                    bool is_screencast,
                    absl::optional<bool> needs_denoising)
      : VideoTrackSource(false),
        capture_adapter_(capture_adapter),
        is_screencast_(is_screencast),
        needs_denoising_(needs_denoising) {}

  WebRtcVideoCapturerAdapter* capture_adapter() const {
    return capture_adapter_.get();
  }

  bool is_screencast() const override { return is_screencast_; }
  absl::optional<bool> needs_denoising() const override {
    return needs_denoising_;
  }

 protected:
  rtc::VideoSourceInterface<webrtc::VideoFrame>* source() override {
    return capture_adapter_.get();
  }

 private:
  std::unique_ptr<WebRtcVideoCapturerAdapter> const capture_adapter_;
  const bool is_screencast_;
  const absl::optional<bool> needs_denoising_;
};

namespace {

// The default number of microseconds that should elapse since the last video
// frame was received, before requesting a refresh frame.
const int64_t kDefaultRefreshIntervalMicros =
    base::Time::kMicrosecondsPerSecond;

// A lower-bound for the refresh interval.
const int64_t kLowerBoundRefreshIntervalMicros =
    base::Time::kMicrosecondsPerSecond / media::limits::kMaxFramesPerSecond;

webrtc::VideoTrackInterface::ContentHint ContentHintTypeToWebRtcContentHint(
    blink::WebMediaStreamTrack::ContentHintType content_hint) {
  switch (content_hint) {
    case blink::WebMediaStreamTrack::ContentHintType::kNone:
      return webrtc::VideoTrackInterface::ContentHint::kNone;
    case blink::WebMediaStreamTrack::ContentHintType::kAudioSpeech:
    case blink::WebMediaStreamTrack::ContentHintType::kAudioMusic:
      NOTREACHED();
      break;
    case blink::WebMediaStreamTrack::ContentHintType::kVideoMotion:
      return webrtc::VideoTrackInterface::ContentHint::kFluid;
    case blink::WebMediaStreamTrack::ContentHintType::kVideoDetail:
      return webrtc::VideoTrackInterface::ContentHint::kDetailed;
    case blink::WebMediaStreamTrack::ContentHintType::kVideoText:
      return webrtc::VideoTrackInterface::ContentHint::kText;
  }
  NOTREACHED();
  return webrtc::VideoTrackInterface::ContentHint::kNone;
}

}  // namespace

// Simple help class used for receiving video frames on the IO-thread from a
// MediaStreamVideoTrack and forward the frames to a WebRtcVideoCapturerAdapter
// on libjingle's worker thread. WebRtcVideoCapturerAdapter implements a video
// capturer for libjingle.
class MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter
    : public base::RefCountedThreadSafe<WebRtcVideoSourceAdapter> {
 public:
  WebRtcVideoSourceAdapter(
      const scoped_refptr<base::SingleThreadTaskRunner>&
          libjingle_worker_thread,
      const scoped_refptr<WebRtcVideoSource>& source,
      base::TimeDelta refresh_interval,
      const base::RepeatingClosure& refresh_callback,
      scoped_refptr<base::SingleThreadTaskRunner> task_runner);

  // MediaStreamVideoWebRtcSink can be destroyed on the main render thread or
  // libjingles worker thread since it posts video frames on that thread. But
  // |video_source_| must be released on the main render thread before the
  // PeerConnectionFactory has been destroyed. The only way to ensure that is to
  // make sure |video_source_| is released when MediaStreamVideoWebRtcSink() is
  // destroyed.
  void ReleaseSourceOnMainThread();

  void OnVideoFrameOnIO(const scoped_refptr<media::VideoFrame>& frame,
                        base::TimeTicks estimated_capture_time);

 private:
  friend class base::RefCountedThreadSafe<WebRtcVideoSourceAdapter>;

  void OnVideoFrameOnWorkerThread(
      const scoped_refptr<media::VideoFrame>& frame);

  virtual ~WebRtcVideoSourceAdapter();

  // Called whenever a video frame was just delivered on the IO thread. This
  // restarts the delay period before the |refresh_timer_| will fire the next
  // time.
  void ResetRefreshTimerOnMainThread();

  scoped_refptr<base::SingleThreadTaskRunner> render_task_runner_;

  // |render_thread_checker_| is bound to the main render thread.
  base::ThreadChecker render_thread_checker_;
  // Used to DCHECK that frames are called on the IO-thread.
  base::ThreadChecker io_thread_checker_;

  // Used for posting frames to libjingle's worker thread. Accessed on the
  // IO-thread.
  scoped_refptr<base::SingleThreadTaskRunner> libjingle_worker_thread_;

  scoped_refptr<webrtc::VideoTrackSourceInterface> video_source_;

  // Used to protect |capture_adapter_|. It is taken by libjingle's worker
  // thread for each video frame that is delivered but only taken on the
  // main render thread in ReleaseSourceOnMainThread() when
  // the owning MediaStreamVideoWebRtcSink is being destroyed.
  base::Lock capture_adapter_stop_lock_;
  // |capture_adapter_| is owned by |video_source_|
  WebRtcVideoCapturerAdapter* capture_adapter_;

  // Requests a refresh frame at regular intervals. The delay on this timer is
  // reset each time a frame is received so that it will not fire for at least
  // an additional period. This means refresh frames will only be requested when
  // the source has halted delivery (e.g., a screen capturer stops sending
  // frames because the screen is not being updated).
  //
  // This mechanism solves a number of problems. First, it will ensure that
  // remote clients that join a distributed session receive a first video frame
  // in a timely manner. Second, it will allow WebRTC's internal bandwidth
  // estimation logic to maintain a more optimal state, since sending a video
  // frame will "prime it." Third, it allows lossy encoders to clean up
  // artifacts in a still image.  http://crbug.com/486274
  base::RepeatingTimer refresh_timer_;
};

MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter::WebRtcVideoSourceAdapter(
    const scoped_refptr<base::SingleThreadTaskRunner>& libjingle_worker_thread,
    const scoped_refptr<WebRtcVideoSource>& source,
    base::TimeDelta refresh_interval,
    const base::RepeatingClosure& refresh_callback,
    scoped_refptr<base::SingleThreadTaskRunner> task_runner)
    : render_task_runner_(std::move(task_runner)),
      libjingle_worker_thread_(libjingle_worker_thread),
      video_source_(source),
      capture_adapter_(source->capture_adapter()) {
  DCHECK(render_task_runner_->RunsTasksInCurrentSequence());
  io_thread_checker_.DetachFromThread();
  if (!refresh_interval.is_zero()) {
    VLOG(1) << "Starting frame refresh timer with interval "
            << refresh_interval.InMillisecondsF() << " ms.";
    refresh_timer_.Start(FROM_HERE, refresh_interval, refresh_callback);
  }
}

MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter::
    ~WebRtcVideoSourceAdapter() {
  DVLOG(3) << "~WebRtcVideoSourceAdapter()";
  DCHECK(!capture_adapter_);
  // This object can be destroyed on the main render thread or libjingles worker
  // thread since it posts video frames on that thread. But |video_source_| must
  // be released on the main render thread before the PeerConnectionFactory has
  // been destroyed. The only way to ensure that is to make sure |video_source_|
  // is released when MediaStreamVideoWebRtcSink() is destroyed.
}

void MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter::
    ResetRefreshTimerOnMainThread() {
  DCHECK(render_thread_checker_.CalledOnValidThread());
  if (refresh_timer_.IsRunning())
    refresh_timer_.Reset();
}

void MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter::
    ReleaseSourceOnMainThread() {
  DCHECK(render_thread_checker_.CalledOnValidThread());
  // Since frames are posted to the worker thread, this object might be deleted
  // on that thread. However, since |video_source_| was created on the render
  // thread, it should be released on the render thread.
  base::AutoLock auto_lock(capture_adapter_stop_lock_);
  // |video_source| owns |capture_adapter_|.
  capture_adapter_ = nullptr;
  video_source_ = nullptr;
}

void MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter::OnVideoFrameOnIO(
    const scoped_refptr<media::VideoFrame>& frame,
    base::TimeTicks estimated_capture_time) {
  DCHECK(io_thread_checker_.CalledOnValidThread());
  render_task_runner_->PostTask(
      FROM_HERE,
      base::BindOnce(&WebRtcVideoSourceAdapter::ResetRefreshTimerOnMainThread,
                     this));
  libjingle_worker_thread_->PostTask(
      FROM_HERE,
      base::BindOnce(&WebRtcVideoSourceAdapter::OnVideoFrameOnWorkerThread,
                     this, frame));
}

void MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter::
    OnVideoFrameOnWorkerThread(const scoped_refptr<media::VideoFrame>& frame) {
  DCHECK(libjingle_worker_thread_->BelongsToCurrentThread());
  base::AutoLock auto_lock(capture_adapter_stop_lock_);
  if (capture_adapter_)
    capture_adapter_->OnFrameCaptured(frame);
}

MediaStreamVideoWebRtcSink::MediaStreamVideoWebRtcSink(
    const blink::WebMediaStreamTrack& track,
    PeerConnectionDependencyFactory* factory,
    scoped_refptr<base::SingleThreadTaskRunner> task_runner)
    : weak_factory_(this) {
  MediaStreamVideoTrack* video_track =
      MediaStreamVideoTrack::GetVideoTrack(track);
  DCHECK(video_track);

  absl::optional<bool> needs_denoising =
      ToAbslOptionalBool(video_track->noise_reduction());

  bool is_screencast = video_track->is_screencast();
  base::Optional<double> min_frame_rate = video_track->min_frame_rate();
  base::Optional<double> max_frame_rate = video_track->max_frame_rate();

  // Enable automatic frame refreshes for the screen capture sources, which will
  // stop producing frames whenever screen content is not changing. Check the
  // frameRate constraint to determine the rate of refreshes. If a minimum
  // frameRate is provided, use that. Otherwise, use the maximum frameRate if it
  // happens to be less than the default.
  base::TimeDelta refresh_interval = base::TimeDelta::FromMicroseconds(0);
  if (is_screencast) {
    // Start with the default refresh interval, and refine based on constraints.
    refresh_interval =
        base::TimeDelta::FromMicroseconds(kDefaultRefreshIntervalMicros);
    if (min_frame_rate.has_value()) {
      refresh_interval =
          base::TimeDelta::FromMicroseconds(base::saturated_cast<int64_t>(
              base::Time::kMicrosecondsPerSecond / *min_frame_rate));
    }
    if (max_frame_rate.has_value()) {
      const base::TimeDelta alternate_refresh_interval =
          base::TimeDelta::FromMicroseconds(base::saturated_cast<int64_t>(
              base::Time::kMicrosecondsPerSecond / *max_frame_rate));
      refresh_interval = std::max(refresh_interval, alternate_refresh_interval);
    }
    if (refresh_interval.InMicroseconds() < kLowerBoundRefreshIntervalMicros) {
      refresh_interval =
          base::TimeDelta::FromMicroseconds(kLowerBoundRefreshIntervalMicros);
    }
  }

  // TODO(pbos): Consolidate WebRtcVideoCapturerAdapter into WebRtcVideoSource
  // by removing the need for and dependency on a cricket::VideoCapturer.
  video_source_ = scoped_refptr<WebRtcVideoSource>(
      new rtc::RefCountedObject<WebRtcVideoSource>(
          new WebRtcVideoCapturerAdapter(is_screencast), is_screencast,
          needs_denoising));

  // TODO(pbos): Consolidate the local video track with the source proxy and
  // move into PeerConnectionDependencyFactory. This now separately holds on a
  // reference to the proxy object because
  // PeerConnectionFactory::CreateVideoTrack doesn't do reference counting.
  video_source_proxy_ =
      factory->CreateVideoTrackSourceProxy(video_source_.get());
  video_track_ = factory->CreateLocalVideoTrack(track.Id().Utf8(),
                                                video_source_proxy_.get());

  video_track_->set_content_hint(
      ContentHintTypeToWebRtcContentHint(track.ContentHint()));
  video_track_->set_enabled(track.IsEnabled());

  source_adapter_ = new WebRtcVideoSourceAdapter(
      factory->GetWebRtcWorkerThread(), video_source_.get(), refresh_interval,
      base::Bind(&MediaStreamVideoWebRtcSink::RequestRefreshFrame,
                 weak_factory_.GetWeakPtr()),
      std::move(task_runner));

  MediaStreamVideoSink::ConnectToTrack(
      track,
      base::Bind(&WebRtcVideoSourceAdapter::OnVideoFrameOnIO, source_adapter_),
      false);

  DVLOG(3) << "MediaStreamVideoWebRtcSink ctor() : is_screencast "
           << is_screencast;
}

MediaStreamVideoWebRtcSink::~MediaStreamVideoWebRtcSink() {
  DCHECK(thread_checker_.CalledOnValidThread());
  DVLOG(3) << "MediaStreamVideoWebRtcSink dtor().";
  weak_factory_.InvalidateWeakPtrs();
  MediaStreamVideoSink::DisconnectFromTrack();
  source_adapter_->ReleaseSourceOnMainThread();
}

void MediaStreamVideoWebRtcSink::OnEnabledChanged(bool enabled) {
  DCHECK(thread_checker_.CalledOnValidThread());
  video_track_->set_enabled(enabled);
}

void MediaStreamVideoWebRtcSink::OnContentHintChanged(
    blink::WebMediaStreamTrack::ContentHintType content_hint) {
  DCHECK(thread_checker_.CalledOnValidThread());
  video_track_->set_content_hint(
      ContentHintTypeToWebRtcContentHint(content_hint));
}

void MediaStreamVideoWebRtcSink::RequestRefreshFrame() {
  DCHECK(thread_checker_.CalledOnValidThread());
  content::RequestRefreshFrameFromVideoTrack(connected_track());
}

absl::optional<bool>
MediaStreamVideoWebRtcSink::SourceNeedsDenoisingForTesting() const {
  return video_source_->needs_denoising();
}

}  // namespace content