Skip to content

Commit

Permalink
Merge pull request #2266 from freakboy3742/ios-camera
Browse files Browse the repository at this point in the history
Add a cross-platform API for camera access
  • Loading branch information
mhsmith authored Jan 17, 2024
2 parents 8c971a2 + c891f0c commit 126e7e8
Show file tree
Hide file tree
Showing 40 changed files with 1,875 additions and 3 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,7 @@ jobs:
- name: Test App
working-directory: testbed
timeout-minutes: 15
run: ${{ matrix.briefcase-run-prefix }} briefcase run ${{ matrix.backend }} --test ${{ matrix.briefcase-run-args }}

- name: Upload logs
Expand All @@ -254,7 +255,6 @@ jobs:
# only occur in CI, and can't be reproduced locally. When it runs, it will
# open an SSH server (URL reported in the logs) so you can ssh into the CI
# machine.
# - uses: actions/checkout@v3
# - name: Setup tmate session
# uses: mxschmitt/action-tmate@v3
# if: failure()
1 change: 1 addition & 0 deletions changes/2266.feature.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
A cross-platform API for camera access was added.
5 changes: 3 additions & 2 deletions cocoa/src/toga_cocoa/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,8 @@
from .app import App, DocumentApp, MainWindow
from .command import Command
from .documents import Document

# Resources
from .fonts import Font
from .hardware.camera import Camera
from .icons import Icon
from .images import Image
from .paths import Paths
Expand Down Expand Up @@ -52,6 +51,8 @@ def not_implemented(feature):
"Image",
"Paths",
"dialogs",
# Hardware
"Camera",
# Widgets
"ActivityIndicator",
"Box",
Expand Down
301 changes: 301 additions & 0 deletions cocoa/src/toga_cocoa/hardware/camera.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,301 @@
from __future__ import annotations

from threading import Thread

from rubicon.objc import Block, objc_method

import toga
from toga.colors import BLACK, RED
from toga.constants import FlashMode
from toga.style import Pack
from toga.style.pack import COLUMN

# for classes that need to be monkeypatched for testing
from toga_cocoa import libs as cocoa
from toga_cocoa.images import nsdata_to_bytes
from toga_cocoa.libs import (
AVAuthorizationStatus,
AVCaptureFlashMode,
AVCapturePhotoOutput,
AVCaptureSession,
AVCaptureSessionPresetPhoto,
AVCaptureVideoPreviewLayer,
AVLayerVideoGravityResizeAspectFill,
AVMediaTypeVideo,
)


def native_flash_mode(flash):
return {
FlashMode.ON: AVCaptureFlashMode.On,
FlashMode.OFF: AVCaptureFlashMode.Off,
}.get(flash, AVCaptureFlashMode.Auto)


class CameraDevice:
def __init__(self, native):
self.native = native

def id(self):
return str(self.native.uniqueID)

def name(self):
return str(self.native.localizedName)

def has_flash(self):
return self.native.isFlashAvailable()


# This is the native delegate, but we can't force the delegate to be invoked because we
# can't create a mock Photo; so we push all logic to the window, and mark this class no
# cover
class TogaCameraCaptureSession(AVCaptureSession): # pragma: no cover
@objc_method
def captureOutput_didFinishProcessingPhoto_error_(
self, output, photo, error
) -> None:
# A photo has been taken.
self.window.photo_taken(photo)


class TogaCameraWindow(toga.Window):
def __init__(self, camera, device, flash, result):
super().__init__(
title="Camera",
on_close=self.close_window,
resizable=False,
# This size is too small by design; it will be expanded by the layout rules.
size=(640, 360),
)
self.camera = camera
self.result = result

self.create_preview_window()
self.create_camera_session(device, flash)

def create_preview_window(self):
# A preview window, fixed 16:9 aspect ratio
self.preview = toga.Box(style=Pack(width=640, height=360))

# Set an initially empty list of devices. This will be populated once the window
# is shown, so that getting the list of devices doesn't slow down showing the
# capture window.
self.device_select = toga.Selection(
items=[],
on_change=self.change_camera,
style=Pack(width=200),
)

# The shutter button. Initially disabled until we know we have a camera available
self.shutter_button = toga.Button(
icon=toga.Icon("camera", system=True),
on_press=self.take_photo,
style=Pack(background_color=RED),
enabled=False,
)

# The flash mode. Initially disable the flash.
self.flash_mode = toga.Selection(
items=[],
style=Pack(width=75),
)

# Construct the overall layout
self.content = toga.Box(
children=[
# The preview box will have its layer replaced by the the video preview.
# Put the preview box inside another box so that we have a surface that
# can show a black background while the camera is initializing.
toga.Box(
children=[self.preview],
style=Pack(background_color=BLACK),
),
toga.Box(
# Put the controls in a ROW box; the shutter button is
# in the middle, non-flexible, so that it is centered.
children=[
toga.Box(
children=[self.device_select],
style=Pack(flex=1),
),
self.shutter_button,
toga.Box(
children=[
toga.Box(style=Pack(flex=1)),
toga.Label("Flash:"),
self.flash_mode,
],
style=Pack(flex=1),
),
],
style=Pack(padding=10),
),
],
style=Pack(direction=COLUMN),
)

# This is the method that creates the native camera session. Mocking these methods
# is extremely difficult (impossible?); plus we can't know what cameras the test
# machine will have. So - we mock this entire method, and mark it no-cover.
def create_camera_session(self, device, flash): # pragma: no cover
self.camera_session = TogaCameraCaptureSession.alloc().init()
self.camera_session.window = self
self.camera_session.beginConfiguration()

# Create a preview layer, rendering into the preview box
preview_layer = AVCaptureVideoPreviewLayer.layerWithSession(self.camera_session)
preview_layer.setVideoGravity(AVLayerVideoGravityResizeAspectFill)
preview_layer.frame = self.preview._impl.native.bounds
self.preview._impl.native.setLayer(preview_layer)

# Specify that we want photo output.
output = AVCapturePhotoOutput.alloc().init()
output.setHighResolutionCaptureEnabled(True)
self.camera_session.addOutput(output)
self.camera_session.setSessionPreset(AVCaptureSessionPresetPhoto)

# Set a sentinel for the camera input; this won't be set until the user has
# selected a camera (either explicitly or implicitly)
self.camera_input = None

# Apply the configuration
self.camera_session.commitConfiguration()

# Polling camera devices and starting the camera session is a blocking activity.
# Start a background thread to populate the list of camera devices and start the
# camera session.
Thread(
target=self._enable_camera,
kwargs={"device": device, "flash": flash},
).start()

def _enable_camera(self, device, flash):
self.camera_session.startRunning()

# The GUI can only be modified from inside the GUI thread. Add a background task
# to apply the new device list.
self.camera.interface.app.loop.create_task(
self._update_camera_list(toga.App.app.camera.devices, device, flash)
)

async def _update_camera_list(self, devices, device, flash):
self.device_select.items = devices
if device:
self.device_select.value = device

self._update_flash_mode(flash)

def _update_flash_mode(self, flash=FlashMode.AUTO):
if device := self.device_select.value:
if device.has_flash:
self.flash_mode.items = [FlashMode.AUTO, FlashMode.OFF, FlashMode.ON]
self.flash_mode.value = flash
else:
self.flash_mode.items = [FlashMode.OFF]
else:
self.flash_mode.items = []

def change_camera(self, widget=None, **kwargs):
# Remove the existing camera input (if it exists)
for input in self.camera_session.inputs:
self.camera_session.removeInput(input)

if device := self.device_select.value:
input = cocoa.AVCaptureDeviceInput.deviceInputWithDevice(
device._impl.native, error=None
)
self.camera_session.addInput(input)
self.shutter_button.enabled = True
else:
self.shutter_button.enabled = False

self._update_flash_mode()

def close_window(self, widget, **kwargs):
# Stop the camera session
self.camera_session.stopRunning()

# Set the "no result" result
self.result.set_result(None)

# Clear the reference to the preview window, and allow the window to close
self.camera.preview_windows.remove(self)
return True

def take_photo(self, widget, **kwargs):
settings = cocoa.AVCapturePhotoSettings.photoSettings()
settings.flashMode = native_flash_mode(self.flash_mode.value)

self.camera_session.outputs[0].capturePhotoWithSettings(
settings,
delegate=self.camera_session,
)
self.close()

def photo_taken(self, photo):
# Create the result image.
image = toga.Image(nsdata_to_bytes(photo.fileDataRepresentation()))
self.result.set_result(image)

# Stop the camera session
self.camera_session.stopRunning()

# Clear the reference to the preview window.
self.camera.preview_windows.remove(self)


class Camera:
def __init__(self, interface):
self.interface = interface
self.preview_windows = []

def has_permission(self, allow_unknown=False):
# To reset permissions to "factory" status, run:
# tccutil reset Camera
#
# To reset a single app:
# tccutil reset Camera <bundleID>
#
# e.g.
# tccutil reset Camera org.beeware.appname # for a bundled app
# tccutil reset Camera com.microsoft.VSCode # for code running in Visual Studio
# tccutil reset Camera com.apple.Terminal # for code running in the Apple terminal

if allow_unknown:
valid_values = {
AVAuthorizationStatus.Authorized.value,
AVAuthorizationStatus.NotDetermined.value,
}
else:
valid_values = {AVAuthorizationStatus.Authorized.value}

return (
cocoa.AVCaptureDevice.authorizationStatusForMediaType(AVMediaTypeVideo)
in valid_values
)

def request_permission(self, future):
# This block is invoked when the permission is granted; however, permission is
# granted from a different (inaccessible) thread, so it isn't picked up by
# coverage.
def permission_complete(result) -> None: # pragma: no cover
future.set_result(result)

cocoa.AVCaptureDevice.requestAccessForMediaType(
AVMediaTypeVideo,
completionHandler=Block(permission_complete, None, bool),
)

def get_devices(self):
return [
CameraDevice(device)
for device in cocoa.AVCaptureDevice.devicesWithMediaType(AVMediaTypeVideo)
]

def take_photo(self, result, device, flash):
if self.has_permission(allow_unknown=True):
window = TogaCameraWindow(self, device, flash, result)
self.preview_windows.append(window)
window.show()
else:
raise PermissionError("App does not have permission to take photos")
1 change: 1 addition & 0 deletions cocoa/src/toga_cocoa/libs/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
)

from .appkit import * # noqa: F401, F403
from .av_foundation import * # noqa: F401, F403
from .core_graphics import * # noqa: F401, F403
from .core_text import * # noqa: F401, F403
from .foundation import * # noqa: F401, F403
Expand Down
Loading

0 comments on commit 126e7e8

Please sign in to comment.