/* -*- Mode: c++; c-basic-offset: 2; indent-tabs-mode: nil; tab-width: 40 -*- */
/* vim: set ts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "MediaManager.h"
#include "AudioCaptureTrack.h"
#include "AudioDeviceInfo.h"
#include "AudioStreamTrack.h"
#include "CubebDeviceEnumerator.h"
#include "CubebInputStream.h"
#include "MediaTimer.h"
#include "MediaTrackConstraints.h"
#include "MediaTrackGraph.h"
#include "MediaTrackListener.h"
#include "VideoStreamTrack.h"
#include "Tracing.h"
#include "VideoUtils.h"
#include "mozilla/Base64.h"
#include "mozilla/EventTargetCapability.h"
#include "mozilla/MozPromise.h"
#include "mozilla/NullPrincipal.h"
#include "mozilla/PeerIdentity.h"
#include "mozilla/PermissionDelegateHandler.h"
#include "mozilla/Sprintf.h"
#include "mozilla/StaticPrefs_media.h"
#include "mozilla/glean/DomMediaWebrtcMetrics.h"
#include "mozilla/Types.h"
#include "mozilla/dom/BindingDeclarations.h"
#include "mozilla/dom/Document.h"
#include "mozilla/dom/Element.h"
#include "mozilla/dom/FeaturePolicyUtils.h"
#include "mozilla/dom/File.h"
#include "mozilla/dom/GetUserMediaRequestBinding.h"
#include "mozilla/dom/MediaDeviceInfo.h"
#include "mozilla/dom/MediaDevices.h"
#include "mozilla/dom/MediaDevicesBinding.h"
#include "mozilla/dom/MediaStreamBinding.h"
#include "mozilla/dom/MediaStreamTrackBinding.h"
#include "mozilla/dom/Promise.h"
#include "mozilla/dom/UserActivation.h"
#include "mozilla/dom/WindowContext.h"
#include "mozilla/dom/WindowGlobalChild.h"
#include "mozilla/ipc/BackgroundChild.h"
#include "mozilla/ipc/PBackgroundChild.h"
#include "mozilla/media/CamerasTypes.h"
#include "mozilla/media/MediaChild.h"
#include "mozilla/media/MediaTaskUtils.h"
#include "nsAppDirectoryServiceDefs.h"
#include "nsArray.h"
#include "nsContentUtils.h"
#include "nsGlobalWindowInner.h"
#include "nsHashPropertyBag.h"
#include "nsIEventTarget.h"
#include "nsIPermissionManager.h"
#include "nsIUUIDGenerator.h"
#include "nsJSUtils.h"
#include "nsNetCID.h"
#include "nsNetUtil.h"
#include "nsProxyRelease.h"
#include "nspr.h"
#include "nss.h"
#include "pk11pub.h"
/* Using WebRTC backend on Desktops (Mac, Windows, Linux), otherwise default */
#include "MediaEngineFake.h"
#include "MediaEngineSource.h"
#if defined(MOZ_WEBRTC)
# include
"MediaEngineWebRTC.h"
# include
"MediaEngineWebRTCAudio.h"
# include
"browser_logging/WebRtcLog.h"
# include
"modules/audio_processing/include/audio_processing.h"
#endif
#if defined(XP_WIN)
# include <objbase.h>
#endif
// A specialization of nsMainThreadPtrHolder for
// mozilla::dom::CallbackObjectHolder. See documentation for
// nsMainThreadPtrHolder in nsProxyRelease.h. This specialization lets us avoid
// wrapping the CallbackObjectHolder into a separate refcounted object.
template <
class WebIDLCallbackT,
class XPCOMCallbackT>
class nsMainThreadPtrHolder<
mozilla::dom::CallbackObjectHolder<WebIDLCallbackT, XPCOMCallbackT>>
final {
typedef mozilla::dom::CallbackObjectHolder<WebIDLCallbackT, XPCOMCallbackT>
Holder;
public:
nsMainThreadPtrHolder(
const char* aName, Holder&& aHolder)
: mHolder(std::move(aHolder))
#ifndef RELEASE_OR_BETA
,
mName(aName)
#endif
{
MOZ_ASSERT(NS_IsMainThread());
}
private:
// We can be released on any thread.
~nsMainThreadPtrHolder() {
if (NS_IsMainThread()) {
mHolder.Reset();
}
else if (mHolder.GetISupports()) {
nsCOMPtr<nsIEventTarget> target = do_GetMainThread();
MOZ_ASSERT(target);
NS_ProxyRelease(
#ifdef RELEASE_OR_BETA
nullptr,
#else
mName,
#endif
target, mHolder.Forget());
}
}
public:
Holder* get() {
// Nobody should be touching the raw pointer off-main-thread.
if (MOZ_UNLIKELY(!NS_IsMainThread())) {
NS_ERROR(
"Can't dereference nsMainThreadPtrHolder off main thread");
MOZ_CRASH();
}
return &mHolder;
}
bool operator!()
const {
return !mHolder; }
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(nsMainThreadPtrHolder<Holder>)
private:
// Our holder.
Holder mHolder;
#ifndef RELEASE_OR_BETA
const char* mName = nullptr;
#endif
// Copy constructor and operator= not implemented. Once constructed, the
// holder is immutable.
Holder&
operator=(
const nsMainThreadPtrHolder& aOther) =
delete;
nsMainThreadPtrHolder(
const nsMainThreadPtrHolder& aOther) =
delete;
};
namespace mozilla {
LazyLogModule gMediaManagerLog(
"MediaManager");
#define LOG(...) MOZ_LOG(gMediaManagerLog, LogLevel::Debug, (__VA_ARGS__))
class GetUserMediaStreamTask;
class LocalTrackSource;
class SelectAudioOutputTask;
using camera::CamerasAccessStatus;
using dom::BFCacheStatus;
using dom::CallerType;
using dom::ConstrainDOMStringParameters;
using dom::ConstrainDoubleRange;
using dom::ConstrainLongRange;
using dom::DisplayMediaStreamConstraints;
using dom::Document;
using dom::Element;
using dom::FeaturePolicyUtils;
using dom::File;
using dom::GetUserMediaRequest;
using dom::MediaDeviceKind;
using dom::MediaDevices;
using dom::MediaSourceEnum;
using dom::MediaStreamConstraints;
using dom::MediaStreamError;
using dom::MediaStreamTrack;
using dom::MediaStreamTrackSource;
using dom::MediaTrackCapabilities;
using dom::MediaTrackConstraints;
using dom::MediaTrackConstraintSet;
using dom::MediaTrackSettings;
using dom::OwningBooleanOrMediaTrackConstraints;
using dom::OwningStringOrStringSequence;
using dom::OwningStringOrStringSequenceOrConstrainDOMStringParameters;
using dom::Promise;
using dom::Sequence;
using dom::UserActivation;
using dom::WindowGlobalChild;
using ConstDeviceSetPromise = MediaManager::ConstDeviceSetPromise;
using DeviceSetPromise = MediaManager::DeviceSetPromise;
using LocalDevicePromise = MediaManager::LocalDevicePromise;
using LocalDeviceSetPromise = MediaManager::LocalDeviceSetPromise;
using LocalMediaDeviceSetRefCnt = MediaManager::LocalMediaDeviceSetRefCnt;
using MediaDeviceSetRefCnt = MediaManager::MediaDeviceSetRefCnt;
using media::NewRunnableFrom;
using media::NewTaskFrom;
using media::Refcountable;
// Whether main thread actions of MediaManager shutdown (except for clearing
// of sSingleton) have completed.
static bool sHasMainThreadShutdown;
struct DeviceState {
DeviceState(RefPtr<LocalMediaDevice> aDevice,
RefPtr<LocalTrackSource> aTrackSource,
bool aOffWhileDisabled)
: mOffWhileDisabled(aOffWhileDisabled),
mDevice(std::move(aDevice)),
mTrackSource(std::move(aTrackSource)) {
MOZ_ASSERT(mDevice);
MOZ_ASSERT(mTrackSource);
}
// true if we have stopped mDevice, this is a terminal state.
// MainThread only.
bool mStopped =
false;
// true if mDevice is currently enabled.
// A device must be both enabled and unmuted to be turned on and capturing.
// MainThread only.
bool mDeviceEnabled =
false;
// true if mDevice is currently muted.
// A device that is either muted or disabled is turned off and not capturing.
// MainThread only.
bool mDeviceMuted;
// true if the application has currently enabled mDevice.
// MainThread only.
bool mTrackEnabled =
false;
// Time when the application last enabled mDevice.
// MainThread only.
TimeStamp mTrackEnabledTime;
// true if an operation to Start() or Stop() mDevice has been dispatched to
// the media thread and is not finished yet.
// MainThread only.
bool mOperationInProgress =
false;
// true if we are allowed to turn off the underlying source while all tracks
// are disabled. Only affects disabling; always turns off on user-agent mute.
// MainThread only.
bool mOffWhileDisabled =
false;
// Timer triggered by a MediaStreamTrackSource signaling that all tracks got
// disabled. When the timer fires we initiate Stop()ing mDevice.
// If set we allow dynamically stopping and starting mDevice.
// Any thread.
const RefPtr<MediaTimer<TimeStamp>> mDisableTimer =
new MediaTimer<TimeStamp>();
// The underlying device we keep state for. Always non-null.
// Threadsafe access, but see method declarations for individual constraints.
const RefPtr<LocalMediaDevice> mDevice;
// The MediaStreamTrackSource for any tracks (original and clones) originating
// from this device. Always non-null. Threadsafe access, but see method
// declarations for individual constraints.
const RefPtr<LocalTrackSource> mTrackSource;
};
/**
* This mimics the capture state from nsIMediaManagerService.
*/
enum class CaptureState : uint16_t {
Off = nsIMediaManagerService::STATE_NOCAPTURE,
Enabled = nsIMediaManagerService::STATE_CAPTURE_ENABLED,
Disabled = nsIMediaManagerService::STATE_CAPTURE_DISABLED,
};
static CaptureState CombineCaptureState(CaptureState aFirst,
CaptureState aSecond) {
if (aFirst == CaptureState::Enabled || aSecond == CaptureState::Enabled) {
return CaptureState::Enabled;
}
if (aFirst == CaptureState::Disabled || aSecond == CaptureState::Disabled) {
return CaptureState::Disabled;
}
MOZ_ASSERT(aFirst == CaptureState::Off);
MOZ_ASSERT(aSecond == CaptureState::Off);
return CaptureState::Off;
}
static uint16_t FromCaptureState(CaptureState aState) {
MOZ_ASSERT(aState == CaptureState::Off || aState == CaptureState::Enabled ||
aState == CaptureState::Disabled);
return static_cast<uint16_t>(aState);
}
void MediaManager::CallOnError(GetUserMediaErrorCallback& aCallback,
MediaStreamError& aError) {
aCallback.Call(aError);
}
void MediaManager::CallOnSuccess(GetUserMediaSuccessCallback& aCallback,
DOMMediaStream& aStream) {
aCallback.Call(aStream);
}
enum class PersistentPermissionState : uint32_t {
Unknown = nsIPermissionManager::UNKNOWN_ACTION,
Allow = nsIPermissionManager::ALLOW_ACTION,
Deny = nsIPermissionManager::DENY_ACTION,
Prompt = nsIPermissionManager::PROMPT_ACTION,
};
static PersistentPermissionState CheckPermission(
PersistentPermissionState aPermission) {
switch (aPermission) {
case PersistentPermissionState::Unknown:
case PersistentPermissionState::Allow:
case PersistentPermissionState::Deny:
case PersistentPermissionState::Prompt:
return aPermission;
}
MOZ_CRASH(
"Unexpected permission value");
}
struct WindowPersistentPermissionState {
PersistentPermissionState mCameraPermission;
PersistentPermissionState mMicrophonePermission;
};
static Result<WindowPersistentPermissionState, nsresult>
GetPersistentPermissions(uint64_t aWindowId) {
auto* window = nsGlobalWindowInner::GetInnerWindowWithId(aWindowId);
if (NS_WARN_IF(!window) || NS_WARN_IF(!window->GetPrincipal())) {
return Err(NS_ERROR_INVALID_ARG);
}
Document* doc = window->GetExtantDoc();
if (NS_WARN_IF(!doc)) {
return Err(NS_ERROR_INVALID_ARG);
}
nsIPrincipal* principal = window->GetPrincipal();
if (NS_WARN_IF(!principal)) {
return Err(NS_ERROR_INVALID_ARG);
}
nsresult rv;
RefPtr<PermissionDelegateHandler> permDelegate =
doc->GetPermissionDelegateHandler();
if (NS_WARN_IF(!permDelegate)) {
return Err(NS_ERROR_INVALID_ARG);
}
uint32_t audio = nsIPermissionManager::UNKNOWN_ACTION;
uint32_t video = nsIPermissionManager::UNKNOWN_ACTION;
{
rv = permDelegate->GetPermission(
"microphone"_ns, &audio,
true);
if (NS_WARN_IF(NS_FAILED(rv))) {
return Err(rv);
}
rv = permDelegate->GetPermission(
"camera"_ns, &video,
true);
if (NS_WARN_IF(NS_FAILED(rv))) {
return Err(rv);
}
}
return WindowPersistentPermissionState{
CheckPermission(
static_cast<PersistentPermissionState>(video)),
CheckPermission(
static_cast<PersistentPermissionState>(audio))};
}
/**
* DeviceListener has threadsafe refcounting for use across the main, media and
* MTG threads. But it has a non-threadsafe SupportsWeakPtr for WeakPtr usage
* only from main thread, to ensure that garbage- and cycle-collected objects
* don't hold a reference to it during late shutdown.
*/
class DeviceListener :
public SupportsWeakPtr {
public:
typedef MozPromise<
bool /* aIgnored */, RefPtr<MediaMgrError>, true>
DeviceListenerPromise;
NS_INLINE_DECL_THREADSAFE_REFCOUNTING_WITH_DELETE_ON_MAIN_THREAD(
DeviceListener)
DeviceListener();
/**
* Registers this device listener as belonging to the given window listener.
* Stop() must be called on registered DeviceListeners before destruction.
*/
void Register(GetUserMediaWindowListener* aListener);
/**
* Marks this listener as active and creates the internal device state.
*/
void Activate(RefPtr<LocalMediaDevice> aDevice,
RefPtr<LocalTrackSource> aTrackSource,
bool aStartMuted);
/**
* Posts a task to initialize and start the associated device.
*/
RefPtr<DeviceListenerPromise> InitializeAsync();
/**
* Posts a task to stop the device associated with this DeviceListener and
* notifies the associated window listener that a track was stopped.
*
* This will also clean up the weak reference to the associated window
* listener, and tell the window listener to remove its hard reference to this
* DeviceListener, so any caller will need to keep its own hard ref.
*/
void Stop();
/**
* Gets the main thread MediaTrackSettings from the MediaEngineSource
* associated with aTrack.
*/
void GetSettings(MediaTrackSettings& aOutSettings)
const;
/**
* Gets the main thread MediaTrackCapabilities from the MediaEngineSource
* associated with aTrack.
*/
void GetCapabilities(MediaTrackCapabilities& aOutCapabilities)
const;
/**
* Posts a task to set the enabled state of the device associated with this
* DeviceListener to aEnabled and notifies the associated window listener that
* a track's state has changed.
*
* Turning the hardware off while the device is disabled is supported for:
* - Camera (enabled by default, controlled by pref
* "media.getusermedia.camera.off_while_disabled.enabled")
* - Microphone (disabled by default, controlled by pref
* "media.getusermedia.microphone.off_while_disabled.enabled")
* Screen-, app-, or windowsharing is not supported at this time.
*
* The behavior is also different between disabling and enabling a device.
* While enabling is immediate, disabling only happens after a delay.
* This is now defaulting to 3 seconds but can be overriden by prefs:
* - "media.getusermedia.camera.off_while_disabled.delay_ms" and
* - "media.getusermedia.microphone.off_while_disabled.delay_ms".
*
* The delay is in place to prevent misuse by malicious sites. If a track is
* re-enabled before the delay has passed, the device will not be touched
* until another disable followed by the full delay happens.
*/
void SetDeviceEnabled(
bool aEnabled);
/**
* Posts a task to set the muted state of the device associated with this
* DeviceListener to aMuted and notifies the associated window listener that a
* track's state has changed.
*
* Turning the hardware off while the device is muted is supported for:
* - Camera (enabled by default, controlled by pref
* "media.getusermedia.camera.off_while_disabled.enabled")
* - Microphone (disabled by default, controlled by pref
* "media.getusermedia.microphone.off_while_disabled.enabled")
* Screen-, app-, or windowsharing is not supported at this time.
*/
void SetDeviceMuted(
bool aMuted);
/**
* Mutes or unmutes the associated video device if it is a camera.
*/
void MuteOrUnmuteCamera(
bool aMute);
void MuteOrUnmuteMicrophone(
bool aMute);
LocalMediaDevice* GetDevice()
const {
return mDeviceState ? mDeviceState->mDevice.get() : nullptr;
}
bool Activated()
const {
return static_cast<
bool>(mDeviceState); }
bool Stopped()
const {
return mStopped; }
bool CapturingVideo()
const;
bool CapturingAudio()
const;
CaptureState CapturingSource(MediaSourceEnum aSource)
const;
RefPtr<DeviceListenerPromise> ApplyConstraints(
const MediaTrackConstraints& aConstraints, CallerType aCallerType);
PrincipalHandle GetPrincipalHandle()
const;
size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf)
const {
size_t amount = aMallocSizeOf(
this);
// Assume mPrincipalHandle refers to a principal owned elsewhere.
// DeviceState does not have support for memory accounting.
return amount;
}
private:
virtual ~DeviceListener() {
MOZ_ASSERT(mStopped);
MOZ_ASSERT(!mWindowListener);
}
using DeviceOperationPromise =
MozPromise<nsresult,
bool,
/* IsExclusive = */ true>;
/**
* Posts a task to start or stop the device associated with aTrack, based on
* a passed-in boolean. Private method used by SetDeviceEnabled and
* SetDeviceMuted.
*/
RefPtr<DeviceOperationPromise> UpdateDevice(
bool aOn);
// true after this listener has had all devices stopped. MainThread only.
bool mStopped;
// never ever indirect off this; just for assertions
PRThread* mMainThreadCheck;
// Set in Register() on main thread, then read from any thread.
PrincipalHandle mPrincipalHandle;
// Weak pointer to the window listener that owns us. MainThread only.
GetUserMediaWindowListener* mWindowListener;
// Accessed from MediaTrackGraph thread, MediaManager thread, and MainThread
// No locking needed as it's set on Activate() and never assigned to again.
UniquePtr<DeviceState> mDeviceState;
MediaEventListener mCaptureEndedListener;
};
/**
* This class represents a WindowID and handles all MediaTrackListeners
* (here subclassed as DeviceListeners) used to feed GetUserMedia tracks.
* It proxies feedback from them into messages for browser chrome.
* The DeviceListeners are used to Start() and Stop() the underlying
* MediaEngineSource when MediaStreams are assigned and deassigned in content.
*/
class GetUserMediaWindowListener {
friend MediaManager;
public:
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(GetUserMediaWindowListener)
// Create in an inactive state
GetUserMediaWindowListener(uint64_t aWindowID,
const PrincipalHandle& aPrincipalHandle)
: mWindowID(aWindowID),
mPrincipalHandle(aPrincipalHandle),
mChromeNotificationTaskPosted(
false) {}
/**
* Registers an inactive gUM device listener for this WindowListener.
*/
void Register(RefPtr<DeviceListener> aListener) {
MOZ_ASSERT(NS_IsMainThread());
MOZ_ASSERT(aListener);
MOZ_ASSERT(!aListener->Activated());
MOZ_ASSERT(!mInactiveListeners.Contains(aListener),
"Already registered");
MOZ_ASSERT(!mActiveListeners.Contains(aListener),
"Already activated");
aListener->
Register(
this);
mInactiveListeners.AppendElement(std::move(aListener));
}
/**
* Activates an already registered and inactive gUM device listener for this
* WindowListener.
*/
void Activate(RefPtr<DeviceListener> aListener,
RefPtr<LocalMediaDevice> aDevice,
RefPtr<LocalTrackSource> aTrackSource) {
MOZ_ASSERT(NS_IsMainThread());
MOZ_ASSERT(aListener);
MOZ_ASSERT(!aListener->Activated());
MOZ_ASSERT(mInactiveListeners.Contains(aListener),
"Must be registered to activate");
MOZ_ASSERT(!mActiveListeners.Contains(aListener),
"Already activated");
bool muted =
false;
if (aDevice->Kind() == MediaDeviceKind::Videoinput) {
muted = mCamerasAreMuted;
}
else if (aDevice->Kind() == MediaDeviceKind::Audioinput) {
muted = mMicrophonesAreMuted;
}
else {
MOZ_CRASH(
"Unexpected device kind");
}
mInactiveListeners.RemoveElement(aListener);
aListener->Activate(std::move(aDevice), std::move(aTrackSource), muted);
mActiveListeners.AppendElement(std::move(aListener));
}
/**
* Removes all DeviceListeners from this window listener.
* Removes this window listener from the list of active windows, so callers
* need to make sure to hold a strong reference.
*/
void RemoveAll() {
MOZ_ASSERT(NS_IsMainThread());
for (
auto& l : mInactiveListeners.Clone()) {
Remove(l);
}
for (
auto& l : mActiveListeners.Clone()) {
Remove(l);
}
MOZ_ASSERT(mInactiveListeners.Length() == 0);
MOZ_ASSERT(mActiveListeners.Length() == 0);
MediaManager* mgr = MediaManager::GetIfExists();
if (!mgr) {
MOZ_ASSERT(
false,
"MediaManager should stay until everything is removed");
return;
}
GetUserMediaWindowListener* windowListener =
mgr->GetWindowListener(mWindowID);
if (!windowListener) {
nsCOMPtr<nsIObserverService> obs = services::GetObserverService();
auto* globalWindow = nsGlobalWindowInner::GetInnerWindowWithId(mWindowID);
if (globalWindow) {
auto req = MakeRefPtr<GetUserMediaRequest>(
globalWindow, VoidString(), VoidString(),
UserActivation::IsHandlingUserInput());
obs->NotifyWhenScriptSafe(req,
"recording-device-stopped", nullptr);
}
return;
}
MOZ_ASSERT(windowListener ==
this,
"There should only be one window listener per window ID");
LOG(
"GUMWindowListener %p removing windowID %" PRIu64,
this, mWindowID);
mgr->RemoveWindowID(mWindowID);
}
/**
* Removes a listener from our lists. Safe to call without holding a hard
* reference. That said, you'll still want to iterate on a copy of said lists,
* if you end up calling this method (or methods that may call this method) in
* the loop, to avoid inadvertently skipping members.
*
* For use only from GetUserMediaWindowListener and DeviceListener.
*/
bool Remove(RefPtr<DeviceListener> aListener) {
// We refcount aListener on entry since we're going to proxy-release it
// below to prevent the refcount going to zero on callers who might be
// inside the listener, but operating without a hard reference to self.
MOZ_ASSERT(NS_IsMainThread());
if (!mInactiveListeners.RemoveElement(aListener) &&
!mActiveListeners.RemoveElement(aListener)) {
return false;
}
MOZ_ASSERT(!mInactiveListeners.Contains(aListener),
"A DeviceListener should only be once in one of "
"mInactiveListeners and mActiveListeners");
MOZ_ASSERT(!mActiveListeners.Contains(aListener),
"A DeviceListener should only be once in one of "
"mInactiveListeners and mActiveListeners");
LOG(
"GUMWindowListener %p stopping DeviceListener %p.",
this,
aListener.get());
aListener->Stop();
if (LocalMediaDevice* removedDevice = aListener->GetDevice()) {
bool revokePermission =
true;
nsString removedRawId;
nsString removedSourceType;
removedDevice->GetRawId(removedRawId);
removedDevice->GetMediaSource(removedSourceType);
for (
const auto& l : mActiveListeners) {
if (LocalMediaDevice* device = l->GetDevice()) {
nsString rawId;
device->GetRawId(rawId);
if (removedRawId.Equals(rawId)) {
revokePermission =
false;
break;
}
}
}
if (revokePermission) {
nsCOMPtr<nsIObserverService> obs = services::GetObserverService();
auto* window = nsGlobalWindowInner::GetInnerWindowWithId(mWindowID);
auto req = MakeRefPtr<GetUserMediaRequest>(
window, removedRawId, removedSourceType,
UserActivation::IsHandlingUserInput());
obs->NotifyWhenScriptSafe(req,
"recording-device-stopped", nullptr);
}
}
if (mInactiveListeners.Length() == 0 && mActiveListeners.Length() == 0) {
LOG(
"GUMWindowListener %p Removed last DeviceListener. Cleaning up.",
this);
RemoveAll();
}
nsCOMPtr<nsIEventTarget> mainTarget = do_GetMainThread();
// To allow being invoked by callers not holding a strong reference to self,
// hold the listener alive until the stack has unwound, by always
// dispatching a runnable (aAlwaysProxy = true)
NS_ProxyRelease(__func__, mainTarget, aListener.forget(),
true);
return true;
}
/**
* Stops all screen/window/audioCapture sharing, but not camera or microphone.
*/
void StopSharing();
void StopRawID(
const nsString& removedDeviceID);
void MuteOrUnmuteCameras(
bool aMute);
void MuteOrUnmuteMicrophones(
bool aMute);
/**
* Called by one of our DeviceListeners when one of its tracks has changed so
* that chrome state is affected.
* Schedules an event for the next stable state to update chrome.
*/
void ChromeAffectingStateChanged();
/**
* Called in stable state to send a notification to update chrome.
*/
void NotifyChrome();
bool CapturingVideo()
const {
MOZ_ASSERT(NS_IsMainThread());
for (
auto& l : mActiveListeners) {
if (l->CapturingVideo()) {
return true;
}
}
return false;
}
bool CapturingAudio()
const {
MOZ_ASSERT(NS_IsMainThread());
for (
auto& l : mActiveListeners) {
if (l->CapturingAudio()) {
return true;
}
}
return false;
}
CaptureState CapturingSource(MediaSourceEnum aSource)
const {
MOZ_ASSERT(NS_IsMainThread());
CaptureState result = CaptureState::Off;
for (
auto& l : mActiveListeners) {
result = CombineCaptureState(result, l->CapturingSource(aSource));
}
return result;
}
RefPtr<LocalMediaDeviceSetRefCnt> GetDevices() {
RefPtr devices =
new LocalMediaDeviceSetRefCnt();
for (
auto& l : mActiveListeners) {
devices->AppendElement(l->GetDevice());
}
return devices;
}
uint64_t WindowID()
const {
return mWindowID; }
PrincipalHandle GetPrincipalHandle()
const {
return mPrincipalHandle; }
size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf)
const {
size_t amount = aMallocSizeOf(
this);
// Assume mPrincipalHandle refers to a principal owned elsewhere.
amount += mInactiveListeners.ShallowSizeOfExcludingThis(aMallocSizeOf);
for (
const RefPtr<DeviceListener>& listener : mInactiveListeners) {
amount += listener->SizeOfIncludingThis(aMallocSizeOf);
}
amount += mActiveListeners.ShallowSizeOfExcludingThis(aMallocSizeOf);
for (
const RefPtr<DeviceListener>& listener : mActiveListeners) {
amount += listener->SizeOfIncludingThis(aMallocSizeOf);
}
return amount;
}
private:
~GetUserMediaWindowListener() {
MOZ_ASSERT(mInactiveListeners.Length() == 0,
"Inactive listeners should already be removed");
MOZ_ASSERT(mActiveListeners.Length() == 0,
"Active listeners should already be removed");
}
uint64_t mWindowID;
const PrincipalHandle mPrincipalHandle;
// true if we have scheduled a task to notify chrome in the next stable state.
// The task will reset this to false. MainThread only.
bool mChromeNotificationTaskPosted;
nsTArray<RefPtr<DeviceListener>> mInactiveListeners;
nsTArray<RefPtr<DeviceListener>> mActiveListeners;
// Whether camera and microphone access in this window are currently
// User Agent (UA) muted. When true, new and cloned tracks must start
// out muted, to avoid JS circumventing UA mute. Per-camera and
// per-microphone UA muting is not supported.
bool mCamerasAreMuted =
false;
bool mMicrophonesAreMuted =
false;
};
class LocalTrackSource :
public MediaStreamTrackSource {
public:
LocalTrackSource(nsIPrincipal* aPrincipal,
const nsString& aLabel,
const RefPtr<DeviceListener>& aListener,
MediaSourceEnum aSource, MediaTrack* aTrack,
RefPtr<PeerIdentity> aPeerIdentity,
TrackingId aTrackingId = TrackingId())
: MediaStreamTrackSource(aPrincipal, aLabel, std::move(aTrackingId)),
mSource(aSource),
mTrack(aTrack),
mPeerIdentity(std::move(aPeerIdentity)),
mListener(aListener.get()) {}
MediaSourceEnum GetMediaSource()
const override {
return mSource; }
const PeerIdentity* GetPeerIdentity()
const override {
return mPeerIdentity; }
RefPtr<MediaStreamTrackSource::ApplyConstraintsPromise> ApplyConstraints(
const MediaTrackConstraints& aConstraints,
CallerType aCallerType) override {
MOZ_ASSERT(NS_IsMainThread());
if (sHasMainThreadShutdown || !mListener) {
// Track has been stopped, or we are in shutdown. In either case
// there's no observable outcome, so pretend we succeeded.
return MediaStreamTrackSource::ApplyConstraintsPromise::CreateAndResolve(
false, __func__);
}
return mListener->ApplyConstraints(aConstraints, aCallerType);
}
void GetSettings(MediaTrackSettings& aOutSettings) override {
if (mListener) {
mListener->GetSettings(aOutSettings);
}
}
void GetCapabilities(MediaTrackCapabilities& aOutCapabilities) override {
if (mListener) {
mListener->GetCapabilities(aOutCapabilities);
}
}
void Stop() override {
if (mListener) {
mListener->Stop();
mListener = nullptr;
}
if (!mTrack->IsDestroyed()) {
mTrack->Destroy();
}
}
void Disable() override {
if (mListener) {
mListener->SetDeviceEnabled(
false);
}
}
void Enable() override {
if (mListener) {
mListener->SetDeviceEnabled(
true);
}
}
void Mute() {
MutedChanged(
true);
mTrack->SetDisabledTrackMode(DisabledTrackMode::SILENCE_BLACK);
}
void Unmute() {
MutedChanged(
false);
mTrack->SetDisabledTrackMode(DisabledTrackMode::ENABLED);
}
const MediaSourceEnum mSource;
const RefPtr<MediaTrack> mTrack;
const RefPtr<
const PeerIdentity> mPeerIdentity;
protected:
~LocalTrackSource() {
MOZ_ASSERT(NS_IsMainThread());
MOZ_ASSERT(mTrack->IsDestroyed());
}
// This is a weak pointer to avoid having the DeviceListener (which may
// have references to threads and threadpools) kept alive by DOM-objects
// that may have ref-cycles and thus are released very late during
// shutdown, even after xpcom-shutdown-threads. See bug 1351655 for what
// can happen.
WeakPtr<DeviceListener> mListener;
};
class AudioCaptureTrackSource :
public LocalTrackSource {
public:
AudioCaptureTrackSource(nsIPrincipal* aPrincipal, nsPIDOMWindowInner* aWindow,
const nsString& aLabel,
AudioCaptureTrack* aAudioCaptureTrack,
RefPtr<PeerIdentity> aPeerIdentity)
: LocalTrackSource(aPrincipal, aLabel, nullptr,
MediaSourceEnum::AudioCapture, aAudioCaptureTrack,
std::move(aPeerIdentity)),
mWindow(aWindow),
mAudioCaptureTrack(aAudioCaptureTrack) {
mAudioCaptureTrack->Start();
mAudioCaptureTrack->Graph()->RegisterCaptureTrackForWindow(
mWindow->WindowID(), mAudioCaptureTrack);
mWindow->SetAudioCapture(
true);
}
void Stop() override {
MOZ_ASSERT(NS_IsMainThread());
if (!mAudioCaptureTrack->IsDestroyed()) {
MOZ_ASSERT(mWindow);
mWindow->SetAudioCapture(
false);
mAudioCaptureTrack->Graph()->UnregisterCaptureTrackForWindow(
mWindow->WindowID());
mWindow = nullptr;
}
// LocalTrackSource destroys the track.
LocalTrackSource::Stop();
MOZ_ASSERT(mAudioCaptureTrack->IsDestroyed());
}
ProcessedMediaTrack* InputTrack()
const {
return mAudioCaptureTrack.get(); }
protected:
~AudioCaptureTrackSource() {
MOZ_ASSERT(NS_IsMainThread());
MOZ_ASSERT(mAudioCaptureTrack->IsDestroyed());
}
RefPtr<nsPIDOMWindowInner> mWindow;
const RefPtr<AudioCaptureTrack> mAudioCaptureTrack;
};
/**
* nsIMediaDevice implementation.
*/
NS_IMPL_ISUPPORTS(LocalMediaDevice, nsIMediaDevice)
MediaDevice::MediaDevice(MediaEngine* aEngine, MediaSourceEnum aMediaSource,
const nsString& aRawName,
const nsString& aRawID,
const nsString& aRawGroupID, IsScary aIsScary,
const OsPromptable canRequestOsLevelPrompt,
const IsPlaceholder aIsPlaceholder)
: mEngine(aEngine),
mAudioDeviceInfo(nullptr),
mMediaSource(aMediaSource),
mKind(MediaEngineSource::IsVideo(aMediaSource)
? MediaDeviceKind::Videoinput
: MediaDeviceKind::Audioinput),
mScary(aIsScary == IsScary::Yes),
mCanRequestOsLevelPrompt(canRequestOsLevelPrompt == OsPromptable::Yes),
mIsFake(mEngine->IsFake()),
mIsPlaceholder(aIsPlaceholder == IsPlaceholder::Yes),
mType(NS_ConvertASCIItoUTF16(dom::GetEnumString(mKind))),
mRawID(aRawID),
mRawGroupID(aRawGroupID),
mRawName(aRawName) {
MOZ_ASSERT(mEngine);
}
MediaDevice::MediaDevice(MediaEngine* aEngine,
const RefPtr<AudioDeviceInfo>& aAudioDeviceInfo,
const nsString& aRawID)
: mEngine(aEngine),
mAudioDeviceInfo(aAudioDeviceInfo),
mMediaSource(mAudioDeviceInfo->Type() == AudioDeviceInfo::TYPE_INPUT
? MediaSourceEnum::Microphone
: MediaSourceEnum::Other),
mKind(mMediaSource == MediaSourceEnum::Microphone
? MediaDeviceKind::Audioinput
: MediaDeviceKind::Audiooutput),
mScary(
false),
mCanRequestOsLevelPrompt(
false),
mIsFake(
false),
mIsPlaceholder(
false),
mType(NS_ConvertASCIItoUTF16(dom::GetEnumString(mKind))),
mRawID(aRawID),
mRawGroupID(mAudioDeviceInfo->GroupID()),
mRawName(mAudioDeviceInfo->Name()) {}
/* static */
RefPtr<MediaDevice> MediaDevice::CopyWithNewRawGroupId(
const RefPtr<MediaDevice>& aOther,
const nsString& aRawGroupID) {
MOZ_ASSERT(!aOther->mAudioDeviceInfo,
"device not supported");
return new MediaDevice(aOther->mEngine, aOther->mMediaSource,
aOther->mRawName, aOther->mRawID, aRawGroupID,
IsScary(aOther->mScary),
OsPromptable(aOther->mCanRequestOsLevelPrompt),
IsPlaceholder(aOther->mIsPlaceholder));
}
MediaDevice::~MediaDevice() =
default;
LocalMediaDevice::LocalMediaDevice(RefPtr<
const MediaDevice> aRawDevice,
const nsString& aID,
const nsString& aGroupID,
const nsString& aName)
: mRawDevice(std::move(aRawDevice)),
mName(aName),
mID(aID),
mGroupID(aGroupID) {
MOZ_ASSERT(mRawDevice);
}
/**
* Helper functions that implement the constraints algorithm from
* http://dev.w3.org/2011/webrtc/editor/getusermedia.html#methods-5
*/
/* static */
bool LocalMediaDevice::StringsContain(
const OwningStringOrStringSequence& aStrings, nsString aN) {
return aStrings.IsString() ? aStrings.GetAsString() == aN
: aStrings.GetAsStringSequence().Contains(aN);
}
/* static */
uint32_t LocalMediaDevice::FitnessDistance(
nsString aN,
const ConstrainDOMStringParameters& aParams) {
if (aParams.mExact.WasPassed() &&
!StringsContain(aParams.mExact.Value(), aN)) {
return UINT32_MAX;
}
if (aParams.mIdeal.WasPassed() &&
!StringsContain(aParams.mIdeal.Value(), aN)) {
return 1;
}
return 0;
}
// Binding code doesn't templatize well...
/* static */
uint32_t LocalMediaDevice::FitnessDistance(
nsString aN,
const OwningStringOrStringSequenceOrConstrainDOMStringParameters&
aConstraint) {
if (aConstraint.IsString()) {
ConstrainDOMStringParameters params;
params.mIdeal.Construct();
params.mIdeal.Value().SetAsString() = aConstraint.GetAsString();
return FitnessDistance(aN, params);
}
else if (aConstraint.IsStringSequence()) {
ConstrainDOMStringParameters params;
params.mIdeal.Construct();
params.mIdeal.Value().SetAsStringSequence() =
aConstraint.GetAsStringSequence();
return FitnessDistance(aN, params);
}
else {
return FitnessDistance(aN, aConstraint.GetAsConstrainDOMStringParameters());
}
}
uint32_t LocalMediaDevice::GetBestFitnessDistance(
const nsTArray<
const NormalizedConstraintSet*>& aConstraintSets,
CallerType aCallerType) {
MOZ_ASSERT(MediaManager::IsInMediaThread());
MOZ_ASSERT(GetMediaSource() != MediaSourceEnum::Other);
bool isChrome = aCallerType == CallerType::System;
const nsString& id = isChrome ? RawID() : mID;
auto type = GetMediaSource();
uint64_t distance = 0;
if (!aConstraintSets.IsEmpty()) {
if (isChrome
/* For the screen/window sharing preview */ ||
type == MediaSourceEnum::Camera ||
type == MediaSourceEnum::Microphone) {
distance += uint64_t(MediaConstraintsHelper::FitnessDistance(
Some(id), aConstraintSets[0]->mDeviceId)) +
uint64_t(MediaConstraintsHelper::FitnessDistance(
Some(mGroupID), aConstraintSets[0]->mGroupId));
}
}
if (distance < UINT32_MAX) {
// Forward request to underlying object to interrogate per-mode
// capabilities.
distance += Source()->GetBestFitnessDistance(aConstraintSets);
}
return std::min<uint64_t>(distance, UINT32_MAX);
}
NS_IMETHODIMP
LocalMediaDevice::GetRawName(nsAString& aName) {
MOZ_ASSERT(NS_IsMainThread());
aName.Assign(mRawDevice->mRawName);
return NS_OK;
}
NS_IMETHODIMP
LocalMediaDevice::GetType(nsAString& aType) {
MOZ_ASSERT(NS_IsMainThread());
aType.Assign(mRawDevice->mType);
return NS_OK;
}
NS_IMETHODIMP
LocalMediaDevice::GetRawId(nsAString& aID) {
MOZ_ASSERT(NS_IsMainThread());
aID.Assign(RawID());
return NS_OK;
}
NS_IMETHODIMP
LocalMediaDevice::GetId(nsAString& aID) {
MOZ_ASSERT(NS_IsMainThread());
aID.Assign(mID);
return NS_OK;
}
NS_IMETHODIMP
LocalMediaDevice::GetScary(
bool* aScary) {
*aScary = mRawDevice->mScary;
return NS_OK;
}
NS_IMETHODIMP
LocalMediaDevice::GetCanRequestOsLevelPrompt(
bool* aCanRequestOsLevelPrompt) {
*aCanRequestOsLevelPrompt = mRawDevice->mCanRequestOsLevelPrompt;
return NS_OK;
}
void LocalMediaDevice::GetSettings(MediaTrackSettings& aOutSettings) {
MOZ_ASSERT(NS_IsMainThread());
Source()->GetSettings(aOutSettings);
}
void LocalMediaDevice::GetCapabilities(
MediaTrackCapabilities& aOutCapabilities) {
MOZ_ASSERT(NS_IsMainThread());
Source()->GetCapabilities(aOutCapabilities);
}
MediaEngineSource* LocalMediaDevice::Source() {
if (!mSource) {
mSource = mRawDevice->mEngine->CreateSource(mRawDevice);
}
return mSource;
}
const TrackingId& LocalMediaDevice::GetTrackingId()
const {
return mSource->GetTrackingId();
}
// Threadsafe since mKind and mSource are const.
NS_IMETHODIMP
LocalMediaDevice::GetMediaSource(nsAString& aMediaSource) {
if (Kind() == MediaDeviceKind::Audiooutput) {
aMediaSource.Truncate();
}
else {
aMediaSource.AssignASCII(dom::GetEnumString(GetMediaSource()));
}
return NS_OK;
}
nsresult LocalMediaDevice::Allocate(
const MediaTrackConstraints& aConstraints,
const MediaEnginePrefs& aPrefs,
uint64_t aWindowID,
const char** aOutBadConstraint) {
MOZ_ASSERT(MediaManager::IsInMediaThread());
// Mock failure for automated tests.
if (IsFake() && aConstraints.mDeviceId.WasPassed() &&
aConstraints.mDeviceId.Value().IsString() &&
aConstraints.mDeviceId.Value().GetAsString().EqualsASCII(
"bad device")) {
return NS_ERROR_FAILURE;
}
return Source()->Allocate(aConstraints, aPrefs, aWindowID, aOutBadConstraint);
}
void LocalMediaDevice::SetTrack(
const RefPtr<MediaTrack>& aTrack,
const PrincipalHandle& aPrincipalHandle) {
MOZ_ASSERT(MediaManager::IsInMediaThread());
Source()->SetTrack(aTrack, aPrincipalHandle);
}
nsresult LocalMediaDevice::Start() {
MOZ_ASSERT(MediaManager::IsInMediaThread());
MOZ_ASSERT(Source());
return Source()->Start();
}
nsresult LocalMediaDevice::Reconfigure(
const MediaTrackConstraints& aConstraints,
const MediaEnginePrefs& aPrefs,
const char** aOutBadConstraint) {
MOZ_ASSERT(MediaManager::IsInMediaThread());
auto type = GetMediaSource();
if (type == MediaSourceEnum::Camera || type == MediaSourceEnum::Microphone) {
NormalizedConstraints c(aConstraints);
if (MediaConstraintsHelper::FitnessDistance(Some(mID), c.mDeviceId) ==
UINT32_MAX) {
*aOutBadConstraint =
"deviceId";
return NS_ERROR_INVALID_ARG;
}
if (MediaConstraintsHelper::FitnessDistance(Some(mGroupID), c.mGroupId) ==
UINT32_MAX) {
*aOutBadConstraint =
"groupId";
return NS_ERROR_INVALID_ARG;
}
}
return Source()->Reconfigure(aConstraints, aPrefs, aOutBadConstraint);
}
nsresult LocalMediaDevice::FocusOnSelectedSource() {
MOZ_ASSERT(MediaManager::IsInMediaThread());
return Source()->FocusOnSelectedSource();
}
nsresult LocalMediaDevice::Stop() {
MOZ_ASSERT(MediaManager::IsInMediaThread());
MOZ_ASSERT(mSource);
return mSource->Stop();
}
nsresult LocalMediaDevice::Deallocate() {
MOZ_ASSERT(MediaManager::IsInMediaThread());
MOZ_ASSERT(mSource);
return mSource->Deallocate();
}
MediaSourceEnum MediaDevice::GetMediaSource()
const {
return mMediaSource; }
static const MediaTrackConstraints& GetInvariant(
const OwningBooleanOrMediaTrackConstraints& aUnion) {
static const MediaTrackConstraints empty;
return aUnion.IsMediaTrackConstraints() ? aUnion.GetAsMediaTrackConstraints()
: empty;
}
// Source getter returning full list
static void GetMediaDevices(MediaEngine* aEngine, MediaSourceEnum aSrcType,
MediaManager::MediaDeviceSet& aResult,
const char* aMediaDeviceName = nullptr) {
MOZ_ASSERT(MediaManager::IsInMediaThread());
LOG(
"%s: aEngine=%p, aSrcType=%" PRIu8
", aMediaDeviceName=%s", __func__,
aEngine,
static_cast<uint8_t>(aSrcType),
aMediaDeviceName ? aMediaDeviceName :
"null");
nsTArray<RefPtr<MediaDevice>> devices;
aEngine->EnumerateDevices(aSrcType, MediaSinkEnum::Other, &devices);
/*
* We're allowing multiple tabs to access the same camera for parity
* with Chrome. See bug 811757 for some of the issues surrounding
* this decision. To disallow, we'd filter by IsAvailable() as we used
* to.
*/
if (aMediaDeviceName && *aMediaDeviceName) {
for (
auto& device : devices) {
if (device->mRawName.EqualsASCII(aMediaDeviceName)) {
aResult.AppendElement(device);
LOG(
"%s: found aMediaDeviceName=%s", __func__, aMediaDeviceName);
break;
}
}
}
else {
aResult = std::move(devices);
if (MOZ_LOG_TEST(gMediaManagerLog, mozilla::LogLevel::Debug)) {
for (
auto& device : aResult) {
LOG(
"%s: appending device=%s", __func__,
NS_ConvertUTF16toUTF8(device->mRawName).get());
}
}
}
}
RefPtr<LocalDeviceSetPromise> MediaManager::SelectSettings(
const MediaStreamConstraints& aConstraints, CallerType aCallerType,
RefPtr<LocalMediaDeviceSetRefCnt> aDevices) {
MOZ_ASSERT(NS_IsMainThread());
// Algorithm accesses device capabilities code and must run on media thread.
// Modifies passed-in aDevices.
return MediaManager::Dispatch<LocalDeviceSetPromise>(
__func__, [aConstraints, devices = std::move(aDevices),
aCallerType](MozPromiseHolder<LocalDeviceSetPromise>& holder) {
auto& devicesRef = *devices;
// Since the advanced part of the constraints algorithm needs to know
// when a candidate set is overconstrained (zero members), we must split
// up the list into videos and audios, and put it back together again at
// the end.
nsTArray<RefPtr<LocalMediaDevice>> videos;
nsTArray<RefPtr<LocalMediaDevice>> audios;
for (
const auto& device : devicesRef) {
MOZ_ASSERT(device->Kind() == MediaDeviceKind::Videoinput ||
device->Kind() == MediaDeviceKind::Audioinput);
if (device->Kind() == MediaDeviceKind::Videoinput) {
videos.AppendElement(device);
}
else if (device->Kind() == MediaDeviceKind::Audioinput) {
audios.AppendElement(device);
}
}
devicesRef.Clear();
const char* badConstraint = nullptr;
bool needVideo = IsOn(aConstraints.mVideo);
bool needAudio = IsOn(aConstraints.mAudio);
if (needVideo && videos.Length()) {
badConstraint = MediaConstraintsHelper::SelectSettings(
NormalizedConstraints(GetInvariant(aConstraints.mVideo)), videos,
aCallerType);
}
if (!badConstraint && needAudio && audios.Length()) {
badConstraint = MediaConstraintsHelper::SelectSettings(
NormalizedConstraints(GetInvariant(aConstraints.mAudio)), audios,
aCallerType);
}
if (badConstraint) {
LOG(
"SelectSettings: bad constraint found! Calling error handler!");
nsString constraint;
constraint.AssignASCII(badConstraint);
holder.Reject(
new MediaMgrError(MediaMgrError::Name::OverconstrainedError,
"",
constraint),
__func__);
return;
}
if (!needVideo == !videos.Length() && !needAudio == !audios.Length()) {
for (
auto& video : videos) {
devicesRef.AppendElement(video);
}
for (
auto& audio : audios) {
devicesRef.AppendElement(audio);
}
}
holder.Resolve(devices, __func__);
});
}
/**
* Describes a requested task that handles response from the UI and sends
* results back to the DOM.
*/
class GetUserMediaTask {
public:
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(GetUserMediaTask)
GetUserMediaTask(uint64_t aWindowID,
const ipc::PrincipalInfo& aPrincipalInfo,
CallerType aCallerType)
: mPrincipalInfo(aPrincipalInfo),
mWindowID(aWindowID),
mCallerType(aCallerType) {}
virtual void Denied(MediaMgrError::Name aName,
const nsCString& aMessage =
""_ns) = 0;
virtual GetUserMediaStreamTask* AsGetUserMediaStreamTask() {
return nullptr; }
virtual SelectAudioOutputTask* AsSelectAudioOutputTask() {
return nullptr; }
uint64_t GetWindowID()
const {
return mWindowID; }
enum CallerType CallerType()
const {
return mCallerType; }
size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf)
const {
size_t amount = aMallocSizeOf(
this);
// Assume mWindowListener is owned by MediaManager.
// Assume mAudioDeviceListener and mVideoDeviceListener are owned by
// mWindowListener.
// Assume PrincipalInfo string buffers are shared.
// Member types without support for accounting of pointees:
// MozPromiseHolder, RefPtr<LocalMediaDevice>.
// We don't have a good way to account for lambda captures for MozPromise
// callbacks.
return amount;
}
protected:
virtual ~GetUserMediaTask() =
default;
// Call GetPrincipalKey again, if not private browing, this time with
// persist = true, to promote deviceIds to persistent, in case they're not
// already. Fire'n'forget.
void PersistPrincipalKey() {
if (IsPrincipalInfoPrivate(mPrincipalInfo)) {
return;
}
media::GetPrincipalKey(mPrincipalInfo,
true)
->Then(
GetCurrentSerialEventTarget(), __func__,
[](
const media::PrincipalKeyPromise::ResolveOrRejectValue& aValue) {
if (aValue.IsReject()) {
LOG(
"Failed get Principal key. Persisting of deviceIds "
"will be broken");
}
});
}
private:
// Thread-safe (object) principal of Window with ID mWindowID
const ipc::PrincipalInfo mPrincipalInfo;
protected:
// The ID of the not-necessarily-toplevel inner Window relevant global
// object of the MediaDevices on which getUserMedia() was called
const uint64_t mWindowID;
// Whether the JS caller of getUserMedia() has system (subject) principal
const enum CallerType mCallerType;
};
/**
* Describes a requested task that handles response from the UI to a
* getUserMedia() request and sends results back to content. If the request
* is allowed and device initialization succeeds, then the MozPromise is
* resolved with a DOMMediaStream having a track or tracks for the approved
* device or devices.
*/
class GetUserMediaStreamTask final :
public GetUserMediaTask {
public:
GetUserMediaStreamTask(
const MediaStreamConstraints& aConstraints,
MozPromiseHolder<MediaManager::StreamPromise>&& aHolder,
uint64_t aWindowID, RefPtr<GetUserMediaWindowListener> aWindowListener,
RefPtr<DeviceListener> aAudioDeviceListener,
RefPtr<DeviceListener> aVideoDeviceListener,
const MediaEnginePrefs& aPrefs,
const ipc::PrincipalInfo& aPrincipalInfo,
enum CallerType aCallerType,
bool aShouldFocusSource)
: GetUserMediaTask(aWindowID, aPrincipalInfo, aCallerType),
mConstraints(aConstraints),
mHolder(std::move(aHolder)),
mWindowListener(std::move(aWindowListener)),
mAudioDeviceListener(std::move(aAudioDeviceListener)),
mVideoDeviceListener(std::move(aVideoDeviceListener)),
mPrefs(aPrefs),
mShouldFocusSource(aShouldFocusSource),
mManager(MediaManager::GetInstance()) {}
void Allowed(RefPtr<LocalMediaDevice> aAudioDevice,
RefPtr<LocalMediaDevice> aVideoDevice) {
MOZ_ASSERT(aAudioDevice || aVideoDevice);
mAudioDevice = std::move(aAudioDevice);
mVideoDevice = std::move(aVideoDevice);
// Reuse the same thread to save memory.
MediaManager::Dispatch(
NewRunnableMethod(
"GetUserMediaStreamTask::AllocateDevices",
this,
&GetUserMediaStreamTask::AllocateDevices));
}
GetUserMediaStreamTask* AsGetUserMediaStreamTask() override {
return this; }
private:
~GetUserMediaStreamTask() override {
if (!mHolder.IsEmpty()) {
Fail(MediaMgrError::Name::NotAllowedError);
}
}
void Fail(MediaMgrError::Name aName,
const nsCString& aMessage =
""_ns,
const nsString& aConstraint = u
""_ns) {
mHolder.Reject(MakeRefPtr<MediaMgrError>(aName, aMessage, aConstraint),
__func__);
// We add a disabled listener to the StreamListeners array until accepted
// If this was the only active MediaStream, remove the window from the list.
NS_DispatchToMainThread(NS_NewRunnableFunction(
"DeviceListener::Stop",
[audio = mAudioDeviceListener, video = mVideoDeviceListener] {
if (audio) {
audio->Stop();
}
if (video) {
video->Stop();
}
}));
}
/**
* Runs on a separate thread and is responsible for allocating devices.
*
* Do not run this on the main thread.
*/
void AllocateDevices() {
MOZ_ASSERT(!NS_IsMainThread());
LOG(
"GetUserMediaStreamTask::AllocateDevices()");
// Allocate a video or audio device and return a MediaStream via
// PrepareDOMStream().
nsresult rv;
const char* errorMsg = nullptr;
const char* badConstraint = nullptr;
if (mAudioDevice) {
auto& constraints = GetInvariant(mConstraints.mAudio);
rv = mAudioDevice->Allocate(constraints, mPrefs, mWindowID,
&badConstraint);
if (NS_FAILED(rv)) {
errorMsg =
"Failed to allocate audiosource";
if (rv == NS_ERROR_NOT_AVAILABLE && !badConstraint) {
nsTArray<RefPtr<LocalMediaDevice>> devices;
devices.AppendElement(mAudioDevice);
badConstraint = MediaConstraintsHelper::SelectSettings(
NormalizedConstraints(constraints), devices, mCallerType);
}
}
}
if (!errorMsg && mVideoDevice) {
auto& constraints = GetInvariant(mConstraints.mVideo);
rv = mVideoDevice->Allocate(constraints, mPrefs, mWindowID,
&badConstraint);
if (NS_FAILED(rv)) {
errorMsg =
"Failed to allocate videosource";
if (rv == NS_ERROR_NOT_AVAILABLE && !badConstraint) {
nsTArray<RefPtr<LocalMediaDevice>> devices;
devices.AppendElement(mVideoDevice);
badConstraint = MediaConstraintsHelper::SelectSettings(
NormalizedConstraints(constraints), devices, mCallerType);
}
if (mAudioDevice) {
mAudioDevice->Deallocate();
}
}
else {
mVideoTrackingId.emplace(mVideoDevice->GetTrackingId());
}
}
if (errorMsg) {
LOG(
"%s %" PRIu32, errorMsg,
static_cast<uint32_t>(rv));
if (badConstraint) {
Fail(MediaMgrError::Name::OverconstrainedError,
""_ns,
NS_ConvertUTF8toUTF16(badConstraint));
}
else {
Fail(MediaMgrError::Name::NotReadableError, nsCString(errorMsg));
}
NS_DispatchToMainThread(
NS_NewRunnableFunction(
"MediaManager::SendPendingGUMRequest", []() {
if (MediaManager* manager = MediaManager::GetIfExists()) {
manager->SendPendingGUMRequest();
}
}));
return;
}
NS_DispatchToMainThread(
NewRunnableMethod(
"GetUserMediaStreamTask::PrepareDOMStream",
this,
&GetUserMediaStreamTask::PrepareDOMStream));
}
public:
void Denied(MediaMgrError::Name aName,
const nsCString& aMessage) override {
MOZ_ASSERT(NS_IsMainThread());
Fail(aName, aMessage);
}
const MediaStreamConstraints& GetConstraints() {
return mConstraints; }
void PrimeVoiceProcessing() {
mPrimingStream = MakeAndAddRef<PrimingCubebVoiceInputStream>();
mPrimingStream->Init();
}
private:
void PrepareDOMStream();
class PrimingCubebVoiceInputStream {
class Listener final :
public CubebInputStream::Listener {
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Listener, override);
private:
~Listener() =
default;
long DataCallback(
const void*,
long) override {
MOZ_CRASH(
"Unexpected data callback");
}
void StateCallback(cubeb_state) override {}
void DeviceChangedCallback() override {}
};
NS_INLINE_DECL_THREADSAFE_REFCOUNTING_WITH_DELETE_ON_EVENT_TARGET(
PrimingCubebVoiceInputStream, mCubebThread.GetEventTarget())
public:
void Init() {
mCubebThread.GetEventTarget()->Dispatch(
NS_NewRunnableFunction(__func__, [
this, self = RefPtr(
this)] {
mCubebThread.AssertOnCurrentThread();
LOG(
"Priming voice processing with stream %p",
this);
TRACE(
"PrimingCubebVoiceInputStream::Init");
const cubeb_devid default_device = nullptr;
const uint32_t mono = 1;
const uint32_t rate = CubebUtils::PreferredSampleRate(
false);
const bool isVoice =
true;
mCubebStream =
CubebInputStream::Create(default_device, mono, rate, isVoice,
MakeRefPtr<Listener>().get());
}));
}
private:
~PrimingCubebVoiceInputStream() {
mCubebThread.AssertOnCurrentThread();
LOG(
"Releasing primed voice processing stream %p",
this);
mCubebStream = nullptr;
}
const EventTargetCapability<nsISerialEventTarget> mCubebThread =
EventTargetCapability<nsISerialEventTarget>(
TaskQueue::Create(CubebUtils::GetCubebOperationThread(),
"PrimingCubebInputStream::mCubebThread")
.get());
UniquePtr<CubebInputStream> mCubebStream MOZ_GUARDED_BY(mCubebThread);
};
// Constraints derived from those passed to getUserMedia() but adjusted for
// preferences, defaults, and security
const MediaStreamConstraints mConstraints;
MozPromiseHolder<MediaManager::StreamPromise> mHolder;
// GetUserMediaWindowListener with which DeviceListeners are registered
const RefPtr<GetUserMediaWindowListener> mWindowListener;
const RefPtr<DeviceListener> mAudioDeviceListener;
const RefPtr<DeviceListener> mVideoDeviceListener;
// MediaDevices are set when selected and Allowed() by the UI.
RefPtr<LocalMediaDevice> mAudioDevice;
RefPtr<LocalMediaDevice> mVideoDevice;
RefPtr<PrimingCubebVoiceInputStream> mPrimingStream;
// Tracking id unique for a video frame source. Set when the corresponding
// device has been allocated.
Maybe<TrackingId> mVideoTrackingId;
// Copy of MediaManager::mPrefs
const MediaEnginePrefs mPrefs;
// media.getusermedia.window.focus_source.enabled
const bool mShouldFocusSource;
// The MediaManager is referenced at construction so that it won't be
// created after its ShutdownBlocker would run.
const RefPtr<MediaManager> mManager;
};
/**
* Creates a MediaTrack, attaches a listener and resolves a MozPromise to
* provide the stream to the DOM.
*
* All of this must be done on the main thread!
*/
void GetUserMediaStreamTask::PrepareDOMStream() {
MOZ_ASSERT(NS_IsMainThread());
LOG(
"GetUserMediaStreamTask::PrepareDOMStream()");
nsGlobalWindowInner* window =
nsGlobalWindowInner::GetInnerWindowWithId(mWindowID);
// We're on main-thread, and the windowlist can only
// be invalidated from the main-thread (see OnNavigation)
if (!mManager->IsWindowListenerStillActive(mWindowListener)) {
// This window is no longer live. mListener has already been removed.
return;
}
MediaTrackGraph::GraphDriverType graphDriverType =
mAudioDevice ? MediaTrackGraph::AUDIO_THREAD_DRIVER
: MediaTrackGraph::SYSTEM_THREAD_DRIVER;
MediaTrackGraph* mtg = MediaTrackGraph::GetInstance(
graphDriverType, window, MediaTrackGraph::REQUEST_DEFAULT_SAMPLE_RATE,
MediaTrackGraph::DEFAULT_OUTPUT_DEVICE);
auto domStream = MakeRefPtr<DOMMediaStream>(window);
RefPtr<LocalTrackSource> audioTrackSource;
RefPtr<LocalTrackSource> videoTrackSource;
nsCOMPtr<nsIPrincipal> principal;
RefPtr<PeerIdentity> peerIdentity = nullptr;
if (!mConstraints.mPeerIdentity.IsEmpty()) {
peerIdentity =
new PeerIdentity(mConstraints.mPeerIdentity);
principal = NullPrincipal::CreateWithInheritedAttributes(
window->GetExtantDoc()->NodePrincipal());
}
else {
principal = window->GetExtantDoc()->NodePrincipal();
}
RefPtr<GenericNonExclusivePromise> firstFramePromise;
if (mAudioDevice) {
if (mAudioDevice->GetMediaSource() == MediaSourceEnum::AudioCapture) {
// AudioCapture is a special case, here, in the sense that we're not
// really using the audio source and the SourceMediaTrack, which acts
// as placeholders. We re-route a number of tracks internally in the
// MTG and mix them down instead.
NS_WARNING(
"MediaCaptureWindowState doesn't handle "
"MediaSourceEnum::AudioCapture. This must be fixed with UX "
"before shipping.");
auto audioCaptureSource = MakeRefPtr<AudioCaptureTrackSource>(
principal, window, u
"Window audio capture"_ns,
mtg->CreateAudioCaptureTrack(), peerIdentity);
audioTrackSource = audioCaptureSource;
RefPtr<MediaStreamTrack> track =
new dom::AudioStreamTrack(
window, audioCaptureSource->InputTrack(), audioCaptureSource);
domStream->AddTrackInternal(track);
}
else {
const nsString& audioDeviceName = mAudioDevice->mName;
RefPtr<MediaTrack> track;
#ifdef MOZ_WEBRTC
if (mAudioDevice->IsFake()) {
track = mtg->CreateSourceTrack(MediaSegment::AUDIO);
}
else {
track = AudioProcessingTrack::Create(mtg);
track->Suspend();
// Microphone source resumes in SetTrack
}
#else
track = mtg->CreateSourceTrack(MediaSegment::AUDIO);
#endif
audioTrackSource =
new LocalTrackSource(
principal, audioDeviceName, mAudioDeviceListener,
mAudioDevice->GetMediaSource(), track, peerIdentity);
MOZ_ASSERT(MediaManager::IsOn(mConstraints.mAudio));
RefPtr<MediaStreamTrack> domTrack =
new dom::AudioStreamTrack(
window, track, audioTrackSource, dom::MediaStreamTrackState::Live,
false, GetInvariant(mConstraints.mAudio));
domStream->AddTrackInternal(domTrack);
}
}
if (mVideoDevice) {
const nsString& videoDeviceName = mVideoDevice->mName;
RefPtr<MediaTrack> track = mtg->CreateSourceTrack(MediaSegment::VIDEO);
videoTrackSource =
new LocalTrackSource(
principal, videoDeviceName, mVideoDeviceListener,
mVideoDevice->GetMediaSource(), track, peerIdentity, *mVideoTrackingId);
MOZ_ASSERT(MediaManager::IsOn(mConstraints.mVideo));
RefPtr<MediaStreamTrack> domTrack =
new dom::VideoStreamTrack(
window, track, videoTrackSource, dom::MediaStreamTrackState::Live,
false, GetInvariant(mConstraints.mVideo));
domStream->AddTrackInternal(domTrack);
switch (mVideoDevice->GetMediaSource()) {
case MediaSourceEnum::Browser:
case MediaSourceEnum::Screen:
case MediaSourceEnum::Window:
// Wait for first frame for screen-sharing devices, to ensure
// with and height settings are available immediately, to pass wpt.
firstFramePromise = mVideoDevice->Source()->GetFirstFramePromise();
break;
default:
break;
}
}
if (!domStream || (!audioTrackSource && !videoTrackSource) ||
sHasMainThreadShutdown) {
LOG(
"Returning error for getUserMedia() - no stream");
mHolder.Reject(
MakeRefPtr<MediaMgrError>(
MediaMgrError::Name::AbortError,
sHasMainThreadShutdown ?
"In shutdown"_ns :
"No stream."_ns),
__func__);
return;
}
// Activate our device listeners. We'll call Start() on the source when we
// get a callback that the MediaStream has started consuming. The listener
// is freed when the page is invalidated (on navigation or close).
if (mAudioDeviceListener) {
mWindowListener->Activate(mAudioDeviceListener, mAudioDevice,
std::move(audioTrackSource));
}
if (mVideoDeviceListener) {
mWindowListener->Activate(mVideoDeviceListener, mVideoDevice,
std::move(videoTrackSource));
}
// Dispatch to the media thread to ask it to start the sources, because that
// can take a while.
typedef DeviceListener::DeviceListenerPromise PromiseType;
AutoTArray<RefPtr<PromiseType>, 2> promises;
if (mAudioDeviceListener) {
promises.AppendElement(mAudioDeviceListener->InitializeAsync());
}
if (mVideoDeviceListener) {
promises.AppendElement(mVideoDeviceListener->InitializeAsync());
}
PromiseType::All(GetMainThreadSerialEventTarget(), promises)
->Then(
GetMainThreadSerialEventTarget(), __func__,
[manager = mManager, windowListener = mWindowListener,
firstFramePromise] {
LOG(
"GetUserMediaStreamTask::PrepareDOMStream: starting success "
"callback following InitializeAsync()");
// Initiating and starting devices succeeded.
windowListener->ChromeAffectingStateChanged();
manager->SendPendingGUMRequest();
if (!firstFramePromise) {
return DeviceListener::DeviceListenerPromise::CreateAndResolve(
true, __func__);
}
RefPtr<DeviceListener::DeviceListenerPromise> resolvePromise =
firstFramePromise->Then(
GetMainThreadSerialEventTarget(), __func__,
[] {
return DeviceListener::DeviceListenerPromise::
CreateAndResolve(
true, __func__);
},
[](nsresult aError) {
MOZ_ASSERT(NS_FAILED(aError));
if (aError == NS_ERROR_UNEXPECTED) {
return DeviceListener::DeviceListenerPromise::
CreateAndReject(
MakeRefPtr<MediaMgrError>(
MediaMgrError::Name::NotAllowedError),
__func__);
}
MOZ_ASSERT(aError == NS_ERROR_ABORT);
return DeviceListener::DeviceListenerPromise::
CreateAndReject(MakeRefPtr<MediaMgrError>(
MediaMgrError::Name::AbortError,
"In shutdown"),
__func__);
});
return resolvePromise;
},
[audio = mAudioDeviceListener,
video = mVideoDeviceListener](RefPtr<MediaMgrError>&& aError) {
LOG(
"GetUserMediaStreamTask::PrepareDOMStream: starting failure "
"callback following InitializeAsync()");
if (audio) {
audio->Stop();
}
if (video) {
video->Stop();
}
return DeviceListener::DeviceListenerPromise::CreateAndReject(
aError, __func__);
})
->Then(
GetMainThreadSerialEventTarget(), __func__,
[holder = std::move(mHolder), domStream, callerType = mCallerType,
shouldFocus = mShouldFocusSource, videoDevice = mVideoDevice](
--> --------------------
--> maximum size reached
--> --------------------
