| // Copyright 2016 The Chromium Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "modules/imagecapture/ImageCapture.h" |
| |
| #include "bindings/core/v8/CallbackPromiseAdapter.h" |
| #include "bindings/core/v8/ScriptPromiseResolver.h" |
| #include "core/dom/DOMException.h" |
| #include "core/dom/ExceptionCode.h" |
| #include "core/fileapi/Blob.h" |
| #include "core/frame/ImageBitmap.h" |
| #include "modules/EventTargetModules.h" |
| #include "modules/imagecapture/MediaSettingsRange.h" |
| #include "modules/imagecapture/PhotoCapabilities.h" |
| #include "modules/imagecapture/PhotoSettings.h" |
| #include "modules/mediastream/MediaStreamTrack.h" |
| #include "modules/mediastream/MediaTrackCapabilities.h" |
| #include "modules/mediastream/MediaTrackSettings.h" |
| #include "platform/WaitableEvent.h" |
| #include "platform/mojo/MojoHelper.h" |
| #include "public/platform/InterfaceProvider.h" |
| #include "public/platform/Platform.h" |
| #include "public/platform/WebImageCaptureFrameGrabber.h" |
| #include "public/platform/WebMediaStreamTrack.h" |
| #include "wtf/PtrUtil.h" |
| |
| namespace blink { |
| |
| using FillLightMode = media::mojom::blink::FillLightMode; |
| using MeteringMode = media::mojom::blink::MeteringMode; |
| |
| namespace { |
| |
| const char kNoServiceError[] = "ImageCapture service unavailable."; |
| |
| bool trackIsInactive(const MediaStreamTrack& track) { |
| // Spec instructs to return an exception if the Track's readyState() is not |
| // "live". Also reject if the track is disabled or muted. |
| return track.readyState() != "live" || !track.enabled() || track.muted(); |
| } |
| |
| MeteringMode parseMeteringMode(const String& blinkMode) { |
| if (blinkMode == "manual") |
| return MeteringMode::MANUAL; |
| if (blinkMode == "single-shot") |
| return MeteringMode::SINGLE_SHOT; |
| if (blinkMode == "continuous") |
| return MeteringMode::CONTINUOUS; |
| if (blinkMode == "none") |
| return MeteringMode::NONE; |
| NOTREACHED(); |
| return MeteringMode::NONE; |
| } |
| |
| FillLightMode parseFillLightMode(const String& blinkMode) { |
| if (blinkMode == "off") |
| return FillLightMode::OFF; |
| if (blinkMode == "auto") |
| return FillLightMode::AUTO; |
| if (blinkMode == "flash") |
| return FillLightMode::FLASH; |
| NOTREACHED(); |
| return FillLightMode::OFF; |
| } |
| |
| WebString toString(MeteringMode value) { |
| switch (value) { |
| case MeteringMode::NONE: |
| return WebString::fromUTF8("none"); |
| case MeteringMode::MANUAL: |
| return WebString::fromUTF8("manual"); |
| case MeteringMode::SINGLE_SHOT: |
| return WebString::fromUTF8("single-shot"); |
| case MeteringMode::CONTINUOUS: |
| return WebString::fromUTF8("continuous"); |
| default: |
| NOTREACHED() << "Unknown MeteringMode"; |
| } |
| return WebString(); |
| } |
| |
| } // anonymous namespace |
| |
| ImageCapture* ImageCapture::create(ExecutionContext* context, |
| MediaStreamTrack* track, |
| ExceptionState& exceptionState) { |
| if (track->kind() != "video") { |
| exceptionState.throwDOMException( |
| NotSupportedError, |
| "Cannot create an ImageCapturer from a non-video Track."); |
| return nullptr; |
| } |
| |
| return new ImageCapture(context, track); |
| } |
| |
| ImageCapture::~ImageCapture() { |
| DCHECK(!hasEventListeners()); |
| // There should be no more outstanding |m_serviceRequests| at this point |
| // since each of them holds a persistent handle to this object. |
| DCHECK(m_serviceRequests.isEmpty()); |
| } |
| |
| const AtomicString& ImageCapture::interfaceName() const { |
| return EventTargetNames::ImageCapture; |
| } |
| |
| ExecutionContext* ImageCapture::getExecutionContext() const { |
| return ContextLifecycleObserver::getExecutionContext(); |
| } |
| |
| bool ImageCapture::hasPendingActivity() const { |
| return getExecutionContext() && hasEventListeners(); |
| } |
| |
| void ImageCapture::contextDestroyed(ExecutionContext*) { |
| removeAllEventListeners(); |
| m_serviceRequests.clear(); |
| DCHECK(!hasEventListeners()); |
| } |
| |
| ScriptPromise ImageCapture::getPhotoCapabilities(ScriptState* scriptState) { |
| ScriptPromiseResolver* resolver = ScriptPromiseResolver::create(scriptState); |
| ScriptPromise promise = resolver->promise(); |
| |
| if (!m_service) { |
| resolver->reject(DOMException::create(NotFoundError, kNoServiceError)); |
| return promise; |
| } |
| m_serviceRequests.insert(resolver); |
| |
| // m_streamTrack->component()->source()->id() is the renderer "name" of the |
| // camera; |
| // TODO(mcasas) consider sending the security origin as well: |
| // scriptState->getExecutionContext()->getSecurityOrigin()->toString() |
| m_service->GetCapabilities( |
| m_streamTrack->component()->source()->id(), |
| convertToBaseCallback(WTF::bind(&ImageCapture::onPhotoCapabilities, |
| wrapPersistent(this), |
| wrapPersistent(resolver)))); |
| return promise; |
| } |
| |
| ScriptPromise ImageCapture::setOptions(ScriptState* scriptState, |
| const PhotoSettings& photoSettings) { |
| ScriptPromiseResolver* resolver = ScriptPromiseResolver::create(scriptState); |
| ScriptPromise promise = resolver->promise(); |
| |
| if (trackIsInactive(*m_streamTrack)) { |
| resolver->reject(DOMException::create( |
| InvalidStateError, "The associated Track is in an invalid state.")); |
| return promise; |
| } |
| |
| if (!m_service) { |
| resolver->reject(DOMException::create(NotFoundError, kNoServiceError)); |
| return promise; |
| } |
| m_serviceRequests.insert(resolver); |
| |
| // TODO(mcasas): should be using a mojo::StructTraits instead. |
| auto settings = media::mojom::blink::PhotoSettings::New(); |
| |
| settings->has_height = photoSettings.hasImageHeight(); |
| if (settings->has_height) |
| settings->height = photoSettings.imageHeight(); |
| settings->has_width = photoSettings.hasImageWidth(); |
| if (settings->has_width) |
| settings->width = photoSettings.imageWidth(); |
| |
| settings->has_red_eye_reduction = photoSettings.hasRedEyeReduction(); |
| if (settings->has_red_eye_reduction) |
| settings->red_eye_reduction = photoSettings.redEyeReduction(); |
| settings->has_fill_light_mode = photoSettings.hasFillLightMode(); |
| if (settings->has_fill_light_mode) { |
| settings->fill_light_mode = |
| parseFillLightMode(photoSettings.fillLightMode()); |
| } |
| |
| m_service->SetOptions(m_streamTrack->component()->source()->id(), |
| std::move(settings), |
| convertToBaseCallback(WTF::bind( |
| &ImageCapture::onSetOptions, wrapPersistent(this), |
| wrapPersistent(resolver)))); |
| return promise; |
| } |
| |
| ScriptPromise ImageCapture::takePhoto(ScriptState* scriptState) { |
| ScriptPromiseResolver* resolver = ScriptPromiseResolver::create(scriptState); |
| ScriptPromise promise = resolver->promise(); |
| |
| if (trackIsInactive(*m_streamTrack)) { |
| resolver->reject(DOMException::create( |
| InvalidStateError, "The associated Track is in an invalid state.")); |
| return promise; |
| } |
| if (!m_service) { |
| resolver->reject(DOMException::create(NotFoundError, kNoServiceError)); |
| return promise; |
| } |
| |
| m_serviceRequests.insert(resolver); |
| |
| // m_streamTrack->component()->source()->id() is the renderer "name" of the |
| // camera; |
| // TODO(mcasas) consider sending the security origin as well: |
| // scriptState->getExecutionContext()->getSecurityOrigin()->toString() |
| m_service->TakePhoto(m_streamTrack->component()->source()->id(), |
| convertToBaseCallback(WTF::bind( |
| &ImageCapture::onTakePhoto, wrapPersistent(this), |
| wrapPersistent(resolver)))); |
| return promise; |
| } |
| |
| ScriptPromise ImageCapture::grabFrame(ScriptState* scriptState) { |
| ScriptPromiseResolver* resolver = ScriptPromiseResolver::create(scriptState); |
| ScriptPromise promise = resolver->promise(); |
| |
| if (trackIsInactive(*m_streamTrack)) { |
| resolver->reject(DOMException::create( |
| InvalidStateError, "The associated Track is in an invalid state.")); |
| return promise; |
| } |
| |
| // Create |m_frameGrabber| the first time. |
| if (!m_frameGrabber) { |
| m_frameGrabber = |
| WTF::wrapUnique(Platform::current()->createImageCaptureFrameGrabber()); |
| } |
| |
| if (!m_frameGrabber) { |
| resolver->reject(DOMException::create( |
| UnknownError, "Couldn't create platform resources")); |
| return promise; |
| } |
| |
| // The platform does not know about MediaStreamTrack, so we wrap it up. |
| WebMediaStreamTrack track(m_streamTrack->component()); |
| m_frameGrabber->grabFrame( |
| &track, new CallbackPromiseAdapter<ImageBitmap, void>(resolver)); |
| |
| return promise; |
| } |
| |
| MediaTrackCapabilities& ImageCapture::getMediaTrackCapabilities() { |
| return m_capabilities; |
| } |
| |
| // TODO(mcasas): make the implementation fully Spec compliant, see inside the |
| // method, https://crbug.com/700607. |
| void ImageCapture::setMediaTrackConstraints( |
| ScriptPromiseResolver* resolver, |
| const MediaTrackConstraintSet& constraints) { |
| if (!m_service) { |
| resolver->reject(DOMException::create(NotFoundError, kNoServiceError)); |
| return; |
| } |
| m_serviceRequests.insert(resolver); |
| |
| auto settings = media::mojom::blink::PhotoSettings::New(); |
| |
| // TODO(mcasas): support other Mode types beyond simple string i.e. the |
| // equivalents of "sequence<DOMString>"" or "ConstrainDOMStringParameters". |
| settings->has_white_balance_mode = constraints.hasWhiteBalanceMode() && |
| constraints.whiteBalanceMode().isString(); |
| if (settings->has_white_balance_mode) { |
| m_currentConstraints.setWhiteBalanceMode(constraints.whiteBalanceMode()); |
| settings->white_balance_mode = |
| parseMeteringMode(constraints.whiteBalanceMode().getAsString()); |
| } |
| settings->has_exposure_mode = |
| constraints.hasExposureMode() && constraints.exposureMode().isString(); |
| if (settings->has_exposure_mode) { |
| m_currentConstraints.setExposureMode(constraints.exposureMode()); |
| settings->exposure_mode = |
| parseMeteringMode(constraints.exposureMode().getAsString()); |
| } |
| |
| settings->has_focus_mode = |
| constraints.hasFocusMode() && constraints.focusMode().isString(); |
| if (settings->has_focus_mode) { |
| m_currentConstraints.setFocusMode(constraints.focusMode()); |
| settings->focus_mode = |
| parseMeteringMode(constraints.focusMode().getAsString()); |
| } |
| |
| // TODO(mcasas): support ConstrainPoint2DParameters. |
| if (constraints.hasPointsOfInterest() && |
| constraints.pointsOfInterest().isPoint2DSequence()) { |
| for (const auto& point : |
| constraints.pointsOfInterest().getAsPoint2DSequence()) { |
| auto mojoPoint = media::mojom::blink::Point2D::New(); |
| mojoPoint->x = point.x(); |
| mojoPoint->y = point.y(); |
| settings->points_of_interest.push_back(std::move(mojoPoint)); |
| } |
| m_currentConstraints.setPointsOfInterest(constraints.pointsOfInterest()); |
| } |
| |
| // TODO(mcasas): support ConstrainDoubleRange where applicable. |
| settings->has_exposure_compensation = |
| constraints.hasExposureCompensation() && |
| constraints.exposureCompensation().isDouble(); |
| if (settings->has_exposure_compensation) { |
| m_currentConstraints.setExposureCompensation( |
| constraints.exposureCompensation()); |
| settings->exposure_compensation = |
| constraints.exposureCompensation().getAsDouble(); |
| } |
| settings->has_color_temperature = constraints.hasColorTemperature() && |
| constraints.colorTemperature().isDouble(); |
| if (settings->has_color_temperature) { |
| m_currentConstraints.setColorTemperature(constraints.colorTemperature()); |
| settings->color_temperature = constraints.colorTemperature().getAsDouble(); |
| } |
| settings->has_iso = constraints.hasIso() && constraints.iso().isDouble(); |
| if (settings->has_iso) { |
| m_currentConstraints.setIso(constraints.iso()); |
| settings->iso = constraints.iso().getAsDouble(); |
| } |
| |
| settings->has_brightness = |
| constraints.hasBrightness() && constraints.brightness().isDouble(); |
| if (settings->has_brightness) { |
| m_currentConstraints.setBrightness(constraints.brightness()); |
| settings->brightness = constraints.brightness().getAsDouble(); |
| } |
| settings->has_contrast = |
| constraints.hasContrast() && constraints.contrast().isDouble(); |
| if (settings->has_contrast) { |
| m_currentConstraints.setContrast(constraints.contrast()); |
| settings->contrast = constraints.contrast().getAsDouble(); |
| } |
| settings->has_saturation = |
| constraints.hasSaturation() && constraints.saturation().isDouble(); |
| if (settings->has_saturation) { |
| m_currentConstraints.setSaturation(constraints.saturation()); |
| settings->saturation = constraints.saturation().getAsDouble(); |
| } |
| settings->has_sharpness = |
| constraints.hasSharpness() && constraints.sharpness().isDouble(); |
| if (settings->has_sharpness) { |
| m_currentConstraints.setSharpness(constraints.sharpness()); |
| settings->sharpness = constraints.sharpness().getAsDouble(); |
| } |
| |
| settings->has_zoom = constraints.hasZoom() && constraints.zoom().isDouble(); |
| if (settings->has_zoom) { |
| m_currentConstraints.setZoom(constraints.zoom()); |
| settings->zoom = constraints.zoom().getAsDouble(); |
| } |
| |
| // TODO(mcasas): support ConstrainBooleanParameters where applicable. |
| settings->has_torch = |
| constraints.hasTorch() && constraints.torch().isBoolean(); |
| if (settings->has_torch) { |
| m_currentConstraints.setTorch(constraints.torch()); |
| settings->torch = constraints.torch().getAsBoolean(); |
| } |
| |
| m_service->SetOptions(m_streamTrack->component()->source()->id(), |
| std::move(settings), |
| convertToBaseCallback(WTF::bind( |
| &ImageCapture::onSetOptions, wrapPersistent(this), |
| wrapPersistent(resolver)))); |
| } |
| |
| const MediaTrackConstraintSet& ImageCapture::getMediaTrackConstraints() const { |
| return m_currentConstraints; |
| } |
| |
| void ImageCapture::getMediaTrackSettings(MediaTrackSettings& settings) const { |
| if (m_capabilities.hasWhiteBalanceMode()) |
| settings.setWhiteBalanceMode(m_capabilities.whiteBalanceMode()[0]); |
| if (m_capabilities.hasExposureMode()) |
| settings.setExposureMode(m_capabilities.exposureMode()[0]); |
| if (m_capabilities.hasFocusMode()) |
| settings.setFocusMode(m_capabilities.focusMode()[0]); |
| |
| if (!m_currentPointsOfInterest.isEmpty()) |
| settings.setPointsOfInterest(m_currentPointsOfInterest); |
| |
| if (m_capabilities.hasExposureCompensation()) { |
| settings.setExposureCompensation( |
| m_capabilities.exposureCompensation()->current()); |
| } |
| if (m_capabilities.hasColorTemperature()) |
| settings.setColorTemperature(m_capabilities.colorTemperature()->current()); |
| if (m_capabilities.hasIso()) |
| settings.setIso(m_capabilities.iso()->current()); |
| |
| if (m_capabilities.hasBrightness()) |
| settings.setBrightness(m_capabilities.brightness()->current()); |
| if (m_capabilities.hasContrast()) |
| settings.setContrast(m_capabilities.contrast()->current()); |
| if (m_capabilities.hasSaturation()) |
| settings.setSaturation(m_capabilities.saturation()->current()); |
| if (m_capabilities.hasSharpness()) |
| settings.setSharpness(m_capabilities.sharpness()->current()); |
| |
| if (m_capabilities.hasZoom()) |
| settings.setZoom(m_capabilities.zoom()->current()); |
| if (m_capabilities.hasTorch()) |
| settings.setTorch(m_capabilities.torch()); |
| } |
| |
| ImageCapture::ImageCapture(ExecutionContext* context, MediaStreamTrack* track) |
| : ContextLifecycleObserver(context), m_streamTrack(track) { |
| DCHECK(m_streamTrack); |
| DCHECK(!m_service.is_bound()); |
| |
| Platform::current()->interfaceProvider()->getInterface( |
| mojo::MakeRequest(&m_service)); |
| |
| m_service.set_connection_error_handler(convertToBaseCallback(WTF::bind( |
| &ImageCapture::onServiceConnectionError, wrapWeakPersistent(this)))); |
| |
| // Launch a retrieval of the current capabilities, which arrive asynchronously |
| // to avoid blocking the main UI thread. |
| m_service->GetCapabilities( |
| m_streamTrack->component()->source()->id(), |
| convertToBaseCallback(WTF::bind(&ImageCapture::onCapabilitiesUpdate, |
| wrapPersistent(this)))); |
| } |
| |
| void ImageCapture::onPhotoCapabilities( |
| ScriptPromiseResolver* resolver, |
| media::mojom::blink::PhotoCapabilitiesPtr capabilities) { |
| if (!m_serviceRequests.contains(resolver)) |
| return; |
| if (capabilities.is_null()) { |
| resolver->reject(DOMException::create(UnknownError, "platform error")); |
| } else { |
| // Update the local capabilities cache. |
| onCapabilitiesUpdateInternal(*capabilities); |
| |
| PhotoCapabilities* caps = PhotoCapabilities::create(); |
| |
| caps->setRedEyeReduction(capabilities->red_eye_reduction); |
| // TODO(mcasas): Remove the explicit MediaSettingsRange::create() when |
| // mojo::StructTraits supports garbage-collected mappings, |
| // https://crbug.com/700180. |
| caps->setImageHeight( |
| MediaSettingsRange::create(std::move(capabilities->height))); |
| caps->setImageWidth( |
| MediaSettingsRange::create(std::move(capabilities->width))); |
| caps->setFillLightMode(capabilities->fill_light_mode); |
| |
| resolver->resolve(caps); |
| } |
| m_serviceRequests.erase(resolver); |
| } |
| |
| void ImageCapture::onSetOptions(ScriptPromiseResolver* resolver, bool result) { |
| if (!m_serviceRequests.contains(resolver)) |
| return; |
| |
| if (!result) { |
| resolver->reject(DOMException::create(UnknownError, "setOptions failed")); |
| m_serviceRequests.erase(resolver); |
| return; |
| } |
| |
| // Retrieve the current device status after setting the options. |
| m_service->GetCapabilities( |
| m_streamTrack->component()->source()->id(), |
| convertToBaseCallback(WTF::bind(&ImageCapture::onPhotoCapabilities, |
| wrapPersistent(this), |
| wrapPersistent(resolver)))); |
| } |
| |
| void ImageCapture::onTakePhoto(ScriptPromiseResolver* resolver, |
| media::mojom::blink::BlobPtr blob) { |
| if (!m_serviceRequests.contains(resolver)) |
| return; |
| |
| // TODO(mcasas): Should be using a mojo::StructTraits. |
| if (blob->data.isEmpty()) |
| resolver->reject(DOMException::create(UnknownError, "platform error")); |
| else |
| resolver->resolve( |
| Blob::create(blob->data.data(), blob->data.size(), blob->mime_type)); |
| m_serviceRequests.erase(resolver); |
| } |
| |
| void ImageCapture::onCapabilitiesUpdate( |
| media::mojom::blink::PhotoCapabilitiesPtr capabilities) { |
| if (!capabilities.is_null()) |
| onCapabilitiesUpdateInternal(*capabilities); |
| } |
| |
| void ImageCapture::onCapabilitiesUpdateInternal( |
| const media::mojom::blink::PhotoCapabilities& capabilities) { |
| // TODO(mcasas): adapt the mojo interface to return a list of supported Modes |
| // when moving these out of PhotoCapabilities, https://crbug.com/700607. |
| m_capabilities.setWhiteBalanceMode( |
| WTF::Vector<WTF::String>({toString(capabilities.white_balance_mode)})); |
| m_capabilities.setExposureMode( |
| WTF::Vector<WTF::String>({toString(capabilities.exposure_mode)})); |
| m_capabilities.setFocusMode( |
| WTF::Vector<WTF::String>({toString(capabilities.focus_mode)})); |
| |
| if (!capabilities.points_of_interest.isEmpty()) { |
| m_currentPointsOfInterest.clear(); |
| for (const auto& point : capabilities.points_of_interest) { |
| Point2D webPoint; |
| webPoint.setX(point->x); |
| webPoint.setY(point->y); |
| m_currentPointsOfInterest.push_back(mojo::Clone(webPoint)); |
| } |
| } |
| |
| // TODO(mcasas): Remove the explicit MediaSettingsRange::create() when |
| // mojo::StructTraits supports garbage-collected mappings, |
| // https://crbug.com/700180. |
| if (capabilities.exposure_compensation->max != |
| capabilities.exposure_compensation->min) { |
| m_capabilities.setExposureCompensation( |
| MediaSettingsRange::create(*capabilities.exposure_compensation)); |
| } |
| if (capabilities.color_temperature->max != |
| capabilities.color_temperature->min) { |
| m_capabilities.setColorTemperature( |
| MediaSettingsRange::create(*capabilities.color_temperature)); |
| } |
| if (capabilities.iso->max != capabilities.iso->min) |
| m_capabilities.setIso(MediaSettingsRange::create(*capabilities.iso)); |
| |
| if (capabilities.brightness->max != capabilities.brightness->min) { |
| m_capabilities.setBrightness( |
| MediaSettingsRange::create(*capabilities.brightness)); |
| } |
| if (capabilities.contrast->max != capabilities.contrast->min) { |
| m_capabilities.setContrast( |
| MediaSettingsRange::create(*capabilities.contrast)); |
| } |
| if (capabilities.saturation->max != capabilities.saturation->min) { |
| m_capabilities.setSaturation( |
| MediaSettingsRange::create(*capabilities.saturation)); |
| } |
| if (capabilities.sharpness->max != capabilities.sharpness->min) { |
| m_capabilities.setSharpness( |
| MediaSettingsRange::create(*capabilities.sharpness)); |
| } |
| |
| if (capabilities.zoom->max != capabilities.zoom->min) |
| m_capabilities.setZoom(MediaSettingsRange::create(*capabilities.zoom)); |
| |
| m_capabilities.setTorch(capabilities.torch); |
| |
| // TODO(mcasas): do |torch| when the mojom interface is updated, |
| // https://crbug.com/700607. |
| } |
| |
| void ImageCapture::onServiceConnectionError() { |
| m_service.reset(); |
| for (ScriptPromiseResolver* resolver : m_serviceRequests) |
| resolver->reject(DOMException::create(NotFoundError, kNoServiceError)); |
| m_serviceRequests.clear(); |
| } |
| |
| DEFINE_TRACE(ImageCapture) { |
| visitor->trace(m_streamTrack); |
| visitor->trace(m_capabilities); |
| visitor->trace(m_currentConstraints); |
| visitor->trace(m_currentPointsOfInterest); |
| visitor->trace(m_serviceRequests); |
| EventTargetWithInlineData::trace(visitor); |
| ContextLifecycleObserver::trace(visitor); |
| } |
| |
| } // namespace blink |