queue.rs

#[cfg(feature = "trace")]
use crate::device::trace::Action;
use crate::{
    api_log,
    command::{
        extract_texture_selector, validate_linear_texture_data, validate_texture_copy_range,
        ClearError, CommandAllocator, CommandBuffer, CommandEncoderError, CopySide,
        ImageCopyTexture, TransferError,
    },
    conv,
    device::{DeviceError, WaitIdleError},
    get_lowest_common_denom,
    global::Global,
    hal_label,
    id::{self, QueueId},
    init_tracker::{has_copy_partial_init_tracker_coverage, TextureInitRange},
    lock::RwLockWriteGuard,
    resource::{
        Buffer, BufferAccessError, BufferMapState, DestroyedBuffer, DestroyedResourceError,
        DestroyedTexture, FlushedStagingBuffer, InvalidResourceError, Labeled, ParentDevice,
        ResourceErrorIdent, StagingBuffer, Texture, TextureInner, Trackable,
    },
    resource_log,
    track::{self, Tracker, TrackerIndex},
    FastHashMap, SubmissionIndex,
};

use smallvec::SmallVec;

use std::{
    iter,
    mem::{self, ManuallyDrop},
    ptr::NonNull,
    sync::{atomic::Ordering, Arc},
};
use thiserror::Error;

use super::Device;

pub struct Queue {
    raw: ManuallyDrop<Box<dyn hal::DynQueue>>,
    pub(crate) device: Arc<Device>,
}

impl Queue {
    pub(crate) fn new(device: Arc<Device>, raw: Box<dyn hal::DynQueue>) -> Self {
        Queue {
            raw: ManuallyDrop::new(raw),
            device,
        }
    }

    pub(crate) fn raw(&self) -> &dyn hal::DynQueue {
        self.raw.as_ref()
    }
}

crate::impl_resource_type!(Queue);
// TODO: https://github.com/gfx-rs/wgpu/issues/4014
impl Labeled for Queue {
    fn label(&self) -> &str {
        ""
    }
}
crate::impl_parent_device!(Queue);
crate::impl_storage_item!(Queue);

impl Drop for Queue {
    fn drop(&mut self) {
        resource_log!("Drop {}", self.error_ident());
        // SAFETY: we never access `self.raw` beyond this point.
        let queue = unsafe { ManuallyDrop::take(&mut self.raw) };
        self.device.release_queue(queue);
    }
}

#[repr(C)]
pub struct SubmittedWorkDoneClosureC {
    pub callback: unsafe extern "C" fn(user_data: *mut u8),
    pub user_data: *mut u8,
}

#[cfg(send_sync)]
unsafe impl Send for SubmittedWorkDoneClosureC {}

pub struct SubmittedWorkDoneClosure {
    // We wrap this so creating the enum in the C variant can be unsafe,
    // allowing our call function to be safe.
    inner: SubmittedWorkDoneClosureInner,
}

#[cfg(send_sync)]
type SubmittedWorkDoneCallback = Box<dyn FnOnce() + Send + 'static>;
#[cfg(not(send_sync))]
type SubmittedWorkDoneCallback = Box<dyn FnOnce() + 'static>;

enum SubmittedWorkDoneClosureInner {
    Rust { callback: SubmittedWorkDoneCallback },
    C { inner: SubmittedWorkDoneClosureC },
}

impl SubmittedWorkDoneClosure {
    pub fn from_rust(callback: SubmittedWorkDoneCallback) -> Self {
        Self {
            inner: SubmittedWorkDoneClosureInner::Rust { callback },
        }
    }

    /// # Safety
    ///
    /// - The callback pointer must be valid to call with the provided `user_data`
    ///   pointer.
    ///
    /// - Both pointers must point to `'static` data, as the callback may happen at
    ///   an unspecified time.
    pub unsafe fn from_c(inner: SubmittedWorkDoneClosureC) -> Self {
        Self {
            inner: SubmittedWorkDoneClosureInner::C { inner },
        }
    }

    pub(crate) fn call(self) {
        match self.inner {
            SubmittedWorkDoneClosureInner::Rust { callback } => callback(),
            // SAFETY: the contract of the call to from_c says that this unsafe is sound.
            SubmittedWorkDoneClosureInner::C { inner } => unsafe {
                (inner.callback)(inner.user_data)
            },
        }
    }
}

/// A texture or buffer to be freed soon.
///
/// This is just a tagged raw texture or buffer, generally about to be added to
/// some other more specific container like:
///
/// - `PendingWrites::temp_resources`: resources used by queue writes and
///   unmaps, waiting to be folded in with the next queue submission
///
/// - `ActiveSubmission::temp_resources`: temporary resources used by a queue
///   submission, to be freed when it completes
#[derive(Debug)]
pub enum TempResource {
    StagingBuffer(FlushedStagingBuffer),
    DestroyedBuffer(DestroyedBuffer),
    DestroyedTexture(DestroyedTexture),
}

/// A series of raw [`CommandBuffer`]s that have been submitted to a
/// queue, and the [`wgpu_hal::CommandEncoder`] that built them.
///
/// [`CommandBuffer`]: hal::Api::CommandBuffer
/// [`wgpu_hal::CommandEncoder`]: hal::CommandEncoder
pub(crate) struct EncoderInFlight {
    raw: Box<dyn hal::DynCommandEncoder>,
    cmd_buffers: Vec<Box<dyn hal::DynCommandBuffer>>,
    pub(crate) trackers: Tracker,

    /// These are the buffers that have been tracked by `PendingWrites`.
    pub(crate) pending_buffers: FastHashMap<TrackerIndex, Arc<Buffer>>,
    /// These are the textures that have been tracked by `PendingWrites`.
    pub(crate) pending_textures: FastHashMap<TrackerIndex, Arc<Texture>>,
}

impl EncoderInFlight {
    /// Free all of our command buffers.
    ///
    /// Return the command encoder, fully reset and ready to be
    /// reused.
    pub(crate) unsafe fn land(mut self) -> Box<dyn hal::DynCommandEncoder> {
        unsafe { self.raw.reset_all(self.cmd_buffers) };
        {
            // This involves actually decrementing the ref count of all command buffer
            // resources, so can be _very_ expensive.
            profiling::scope!("drop command buffer trackers");
            drop(self.trackers);
            drop(self.pending_buffers);
            drop(self.pending_textures);
        }
        self.raw
    }
}

/// A private command encoder for writes made directly on the device
/// or queue.
///
/// Operations like `buffer_unmap`, `queue_write_buffer`, and
/// `queue_write_texture` need to copy data to the GPU. At the hal
/// level, this must be done by encoding and submitting commands, but
/// these operations are not associated with any specific wgpu command
/// buffer.
///
/// Instead, `Device::pending_writes` owns one of these values, which
/// has its own hal command encoder and resource lists. The commands
/// accumulated here are automatically submitted to the queue the next
/// time the user submits a wgpu command buffer, ahead of the user's
/// commands.
///
/// Important:
/// When locking pending_writes be sure that tracker is not locked
/// and try to lock trackers for the minimum timespan possible
///
/// All uses of [`StagingBuffer`]s end up here.
#[derive(Debug)]
pub(crate) struct PendingWrites {
    pub command_encoder: Box<dyn hal::DynCommandEncoder>,

    /// True if `command_encoder` is in the "recording" state, as
    /// described in the docs for the [`wgpu_hal::CommandEncoder`]
    /// trait.
    ///
    /// [`wgpu_hal::CommandEncoder`]: hal::CommandEncoder
    pub is_recording: bool,

    temp_resources: Vec<TempResource>,
    dst_buffers: FastHashMap<TrackerIndex, Arc<Buffer>>,
    dst_textures: FastHashMap<TrackerIndex, Arc<Texture>>,
}

impl PendingWrites {
    pub fn new(command_encoder: Box<dyn hal::DynCommandEncoder>) -> Self {
        Self {
            command_encoder,
            is_recording: false,
            temp_resources: Vec::new(),
            dst_buffers: FastHashMap::default(),
            dst_textures: FastHashMap::default(),
        }
    }

    pub fn dispose(mut self, device: &dyn hal::DynDevice) {
        unsafe {
            if self.is_recording {
                self.command_encoder.discard_encoding();
            }
            device.destroy_command_encoder(self.command_encoder);
        }

        self.temp_resources.clear();
    }

    pub fn insert_buffer(&mut self, buffer: &Arc<Buffer>) {
        self.dst_buffers
            .insert(buffer.tracker_index(), buffer.clone());
    }

    pub fn insert_texture(&mut self, texture: &Arc<Texture>) {
        self.dst_textures
            .insert(texture.tracker_index(), texture.clone());
    }

    pub fn contains_buffer(&self, buffer: &Arc<Buffer>) -> bool {
        self.dst_buffers.contains_key(&buffer.tracker_index())
    }

    pub fn contains_texture(&self, texture: &Arc<Texture>) -> bool {
        self.dst_textures.contains_key(&texture.tracker_index())
    }

    pub fn consume_temp(&mut self, resource: TempResource) {
        self.temp_resources.push(resource);
    }

    pub fn consume(&mut self, buffer: FlushedStagingBuffer) {
        self.temp_resources
            .push(TempResource::StagingBuffer(buffer));
    }

    fn pre_submit(
        &mut self,
        command_allocator: &CommandAllocator,
        device: &Device,
        queue: &Queue,
    ) -> Result<Option<EncoderInFlight>, DeviceError> {
        if self.is_recording {
            let pending_buffers = mem::take(&mut self.dst_buffers);
            let pending_textures = mem::take(&mut self.dst_textures);

            let cmd_buf = unsafe { self.command_encoder.end_encoding() }
                .map_err(|e| device.handle_hal_error(e))?;
            self.is_recording = false;

            let new_encoder = command_allocator
                .acquire_encoder(device.raw(), queue.raw())
                .map_err(|e| device.handle_hal_error(e))?;

            let encoder = EncoderInFlight {
                raw: mem::replace(&mut self.command_encoder, new_encoder),
                cmd_buffers: vec![cmd_buf],
                trackers: Tracker::new(),
                pending_buffers,
                pending_textures,
            };
            Ok(Some(encoder))
        } else {
            self.dst_buffers.clear();
            self.dst_textures.clear();
            Ok(None)
        }
    }

    pub fn activate(&mut self) -> &mut dyn hal::DynCommandEncoder {
        if !self.is_recording {
            unsafe {
                self.command_encoder
                    .begin_encoding(Some("(wgpu internal) PendingWrites"))
                    .unwrap();
            }
            self.is_recording = true;
        }
        self.command_encoder.as_mut()
    }

    pub fn deactivate(&mut self) {
        if self.is_recording {
            unsafe {
                self.command_encoder.discard_encoding();
            }
            self.is_recording = false;
        }
    }
}

#[derive(Clone, Debug, Error)]
#[non_exhaustive]
pub enum QueueWriteError {
    #[error(transparent)]
    Queue(#[from] DeviceError),
    #[error(transparent)]
    Transfer(#[from] TransferError),
    #[error(transparent)]
    MemoryInitFailure(#[from] ClearError),
    #[error(transparent)]
    DestroyedResource(#[from] DestroyedResourceError),
    #[error(transparent)]
    InvalidResource(#[from] InvalidResourceError),
}

#[derive(Clone, Debug, Error)]
#[non_exhaustive]
pub enum QueueSubmitError {
    #[error(transparent)]
    Queue(#[from] DeviceError),
    #[error(transparent)]
    DestroyedResource(#[from] DestroyedResourceError),
    #[error(transparent)]
    Unmap(#[from] BufferAccessError),
    #[error("{0} is still mapped")]
    BufferStillMapped(ResourceErrorIdent),
    #[error("Surface output was dropped before the command buffer got submitted")]
    SurfaceOutputDropped,
    #[error("Surface was unconfigured before the command buffer got submitted")]
    SurfaceUnconfigured,
    #[error("GPU got stuck :(")]
    StuckGpu,
    #[error(transparent)]
    InvalidResource(#[from] InvalidResourceError),
    #[error(transparent)]
    CommandEncoder(#[from] CommandEncoderError),
}

//TODO: move out common parts of write_xxx.

impl Global {
    pub fn queue_write_buffer(
        &self,
        queue_id: QueueId,
        buffer_id: id::BufferId,
        buffer_offset: wgt::BufferAddress,
        data: &[u8],
    ) -> Result<(), QueueWriteError> {
        profiling::scope!("Queue::write_buffer");
        api_log!("Queue::write_buffer {buffer_id:?} {}bytes", data.len());

        let hub = &self.hub;

        let buffer = hub.buffers.get(buffer_id).get()?;

        let queue = hub.queues.get(queue_id);

        let device = &queue.device;

        let data_size = data.len() as wgt::BufferAddress;

        #[cfg(feature = "trace")]
        if let Some(ref mut trace) = *device.trace.lock() {
            let data_path = trace.make_binary("bin", data);
            trace.add(Action::WriteBuffer {
                id: buffer_id,
                data: data_path,
                range: buffer_offset..buffer_offset + data_size,
                queued: true,
            });
        }

        buffer.same_device_as(queue.as_ref())?;

        let data_size = if let Some(data_size) = wgt::BufferSize::new(data_size) {
            data_size
        } else {
            log::trace!("Ignoring write_buffer of size 0");
            return Ok(());
        };

        // Platform validation requires that the staging buffer always be
        // freed, even if an error occurs. All paths from here must call
        // `device.pending_writes.consume`.
        let mut staging_buffer = StagingBuffer::new(device, data_size)?;
        let mut pending_writes = device.pending_writes.lock();

        let staging_buffer = {
            profiling::scope!("copy");
            staging_buffer.write(data);
            staging_buffer.flush()
        };

        let result = self.queue_write_staging_buffer_impl(
            &queue,
            device,
            &mut pending_writes,
            &staging_buffer,
            buffer_id,
            buffer_offset,
        );

        pending_writes.consume(staging_buffer);
        result
    }

    pub fn queue_create_staging_buffer(
        &self,
        queue_id: QueueId,
        buffer_size: wgt::BufferSize,
        id_in: Option<id::StagingBufferId>,
    ) -> Result<(id::StagingBufferId, NonNull<u8>), QueueWriteError> {
        profiling::scope!("Queue::create_staging_buffer");
        let hub = &self.hub;

        let queue = hub.queues.get(queue_id);

        let device = &queue.device;

        let staging_buffer = StagingBuffer::new(device, buffer_size)?;
        let ptr = unsafe { staging_buffer.ptr() };

        let fid = hub.staging_buffers.prepare(id_in);
        let id = fid.assign(staging_buffer);
        resource_log!("Queue::create_staging_buffer {id:?}");

        Ok((id, ptr))
    }

    pub fn queue_write_staging_buffer(
        &self,
        queue_id: QueueId,
        buffer_id: id::BufferId,
        buffer_offset: wgt::BufferAddress,
        staging_buffer_id: id::StagingBufferId,
    ) -> Result<(), QueueWriteError> {
        profiling::scope!("Queue::write_staging_buffer");
        let hub = &self.hub;

        let queue = hub.queues.get(queue_id);

        let device = &queue.device;

        let staging_buffer = hub.staging_buffers.remove(staging_buffer_id);

        let mut pending_writes = device.pending_writes.lock();

        // At this point, we have taken ownership of the staging_buffer from the
        // user. Platform validation requires that the staging buffer always
        // be freed, even if an error occurs. All paths from here must call
        // `device.pending_writes.consume`.
        let staging_buffer = staging_buffer.flush();

        let result = self.queue_write_staging_buffer_impl(
            &queue,
            device,
            &mut pending_writes,
            &staging_buffer,
            buffer_id,
            buffer_offset,
        );

        pending_writes.consume(staging_buffer);
        result
    }

    pub fn queue_validate_write_buffer(
        &self,
        _queue_id: QueueId,
        buffer_id: id::BufferId,
        buffer_offset: u64,
        buffer_size: wgt::BufferSize,
    ) -> Result<(), QueueWriteError> {
        profiling::scope!("Queue::validate_write_buffer");
        let hub = &self.hub;

        let buffer = hub.buffers.get(buffer_id).get()?;

        self.queue_validate_write_buffer_impl(&buffer, buffer_offset, buffer_size)?;

        Ok(())
    }

    fn queue_validate_write_buffer_impl(
        &self,
        buffer: &Buffer,
        buffer_offset: u64,
        buffer_size: wgt::BufferSize,
    ) -> Result<(), TransferError> {
        buffer.check_usage(wgt::BufferUsages::COPY_DST)?;
        if buffer_size.get() % wgt::COPY_BUFFER_ALIGNMENT != 0 {
            return Err(TransferError::UnalignedCopySize(buffer_size.get()));
        }
        if buffer_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 {
            return Err(TransferError::UnalignedBufferOffset(buffer_offset));
        }
        if buffer_offset + buffer_size.get() > buffer.size {
            return Err(TransferError::BufferOverrun {
                start_offset: buffer_offset,
                end_offset: buffer_offset + buffer_size.get(),
                buffer_size: buffer.size,
                side: CopySide::Destination,
            });
        }

        Ok(())
    }

    fn queue_write_staging_buffer_impl(
        &self,
        queue: &Arc<Queue>,
        device: &Arc<Device>,
        pending_writes: &mut PendingWrites,
        staging_buffer: &FlushedStagingBuffer,
        buffer_id: id::BufferId,
        buffer_offset: u64,
    ) -> Result<(), QueueWriteError> {
        let hub = &self.hub;

        let dst = hub.buffers.get(buffer_id).get()?;

        let transition = {
            let mut trackers = device.trackers.lock();
            trackers.buffers.set_single(&dst, hal::BufferUses::COPY_DST)
        };

        let snatch_guard = device.snatchable_lock.read();
        let dst_raw = dst.try_raw(&snatch_guard)?;

        dst.same_device_as(queue.as_ref())?;

        self.queue_validate_write_buffer_impl(&dst, buffer_offset, staging_buffer.size)?;

        let region = hal::BufferCopy {
            src_offset: 0,
            dst_offset: buffer_offset,
            size: staging_buffer.size,
        };
        let barriers = iter::once(hal::BufferBarrier {
            buffer: staging_buffer.raw(),
            usage: hal::BufferUses::MAP_WRITE..hal::BufferUses::COPY_SRC,
        })
        .chain(transition.map(|pending| pending.into_hal(&dst, &snatch_guard)))
        .collect::<Vec<_>>();
        let encoder = pending_writes.activate();
        unsafe {
            encoder.transition_buffers(&barriers);
            encoder.copy_buffer_to_buffer(staging_buffer.raw(), dst_raw, &[region]);
        }

        pending_writes.insert_buffer(&dst);

        // Ensure the overwritten bytes are marked as initialized so
        // they don't need to be nulled prior to mapping or binding.
        {
            dst.initialization_status
                .write()
                .drain(buffer_offset..(buffer_offset + staging_buffer.size.get()));
        }

        Ok(())
    }

    pub fn queue_write_texture(
        &self,
        queue_id: QueueId,
        destination: &ImageCopyTexture,
        data: &[u8],
        data_layout: &wgt::ImageDataLayout,
        size: &wgt::Extent3d,
    ) -> Result<(), QueueWriteError> {
        profiling::scope!("Queue::write_texture");
        api_log!("Queue::write_texture {:?} {size:?}", destination.texture);

        let hub = &self.hub;

        let queue = hub.queues.get(queue_id);

        let device = &queue.device;

        #[cfg(feature = "trace")]
        if let Some(ref mut trace) = *device.trace.lock() {
            let data_path = trace.make_binary("bin", data);
            trace.add(Action::WriteTexture {
                to: *destination,
                data: data_path,
                layout: *data_layout,
                size: *size,
            });
        }

        if size.width == 0 || size.height == 0 || size.depth_or_array_layers == 0 {
            log::trace!("Ignoring write_texture of size 0");
            return Ok(());
        }

        let dst = hub.textures.get(destination.texture).get()?;

        dst.same_device_as(queue.as_ref())?;

        dst.check_usage(wgt::TextureUsages::COPY_DST)
            .map_err(TransferError::MissingTextureUsage)?;

        // Note: Doing the copy range validation early is important because ensures that the
        // dimensions are not going to cause overflow in other parts of the validation.
        let (hal_copy_size, array_layer_count) =
            validate_texture_copy_range(destination, &dst.desc, CopySide::Destination, size)?;

        let (selector, dst_base) = extract_texture_selector(destination, size, &dst)?;

        if !dst_base.aspect.is_one() {
            return Err(TransferError::CopyAspectNotOne.into());
        }

        if !conv::is_valid_copy_dst_texture_format(dst.desc.format, destination.aspect) {
            return Err(TransferError::CopyToForbiddenTextureFormat {
                format: dst.desc.format,
                aspect: destination.aspect,
            }
            .into());
        }

        // Note: `_source_bytes_per_array_layer` is ignored since we
        // have a staging copy, and it can have a different value.
        let (required_bytes_in_copy, _source_bytes_per_array_layer) = validate_linear_texture_data(
            data_layout,
            dst.desc.format,
            destination.aspect,
            data.len() as wgt::BufferAddress,
            CopySide::Source,
            size,
            false,
        )?;

        if dst.desc.format.is_depth_stencil_format() {
            device
                .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
                .map_err(TransferError::from)?;
        }

        let mut pending_writes = device.pending_writes.lock();
        let encoder = pending_writes.activate();

        // If the copy does not fully cover the layers, we need to initialize to
        // zero *first* as we don't keep track of partial texture layer inits.
        //
        // Strictly speaking we only need to clear the areas of a layer
        // untouched, but this would get increasingly messy.
        let init_layer_range = if dst.desc.dimension == wgt::TextureDimension::D3 {
            // volume textures don't have a layer range as array volumes aren't supported
            0..1
        } else {
            destination.origin.z..destination.origin.z + size.depth_or_array_layers
        };
        let mut dst_initialization_status = dst.initialization_status.write();
        if dst_initialization_status.mips[destination.mip_level as usize]
            .check(init_layer_range.clone())
            .is_some()
        {
            if has_copy_partial_init_tracker_coverage(size, destination.mip_level, &dst.desc) {
                for layer_range in dst_initialization_status.mips[destination.mip_level as usize]
                    .drain(init_layer_range)
                    .collect::<Vec<std::ops::Range<u32>>>()
                {
                    let mut trackers = device.trackers.lock();
                    crate::command::clear_texture(
                        &dst,
                        TextureInitRange {
                            mip_range: destination.mip_level..(destination.mip_level + 1),
                            layer_range,
                        },
                        encoder,
                        &mut trackers.textures,
                        &device.alignments,
                        device.zero_buffer.as_ref(),
                        &device.snatchable_lock.read(),
                    )
                    .map_err(QueueWriteError::from)?;
                }
            } else {
                dst_initialization_status.mips[destination.mip_level as usize]
                    .drain(init_layer_range);
            }
        }

        let snatch_guard = device.snatchable_lock.read();

        let dst_raw = dst.try_raw(&snatch_guard)?;

        let (block_width, block_height) = dst.desc.format.block_dimensions();
        let width_in_blocks = size.width / block_width;
        let height_in_blocks = size.height / block_height;

        let block_size = dst
            .desc
            .format
            .block_copy_size(Some(destination.aspect))
            .unwrap();
        let bytes_in_last_row = width_in_blocks * block_size;

        let bytes_per_row = data_layout.bytes_per_row.unwrap_or(bytes_in_last_row);
        let rows_per_image = data_layout.rows_per_image.unwrap_or(height_in_blocks);

        let bytes_per_row_alignment =
            get_lowest_common_denom(device.alignments.buffer_copy_pitch.get() as u32, block_size);
        let stage_bytes_per_row = wgt::math::align_to(bytes_in_last_row, bytes_per_row_alignment);

        // Platform validation requires that the staging buffer always be
        // freed, even if an error occurs. All paths from here must call
        // `device.pending_writes.consume`.
        let staging_buffer = if stage_bytes_per_row == bytes_per_row {
            profiling::scope!("copy aligned");
            // Fast path if the data is already being aligned optimally.
            let stage_size = wgt::BufferSize::new(required_bytes_in_copy).unwrap();
            let mut staging_buffer = StagingBuffer::new(device, stage_size)?;
            staging_buffer.write(&data[data_layout.offset as usize..]);
            staging_buffer
        } else {
            profiling::scope!("copy chunked");
            // Copy row by row into the optimal alignment.
            let block_rows_in_copy =
                (size.depth_or_array_layers - 1) * rows_per_image + height_in_blocks;
            let stage_size =
                wgt::BufferSize::new(stage_bytes_per_row as u64 * block_rows_in_copy as u64)
                    .unwrap();
            let mut staging_buffer = StagingBuffer::new(device, stage_size)?;
            let copy_bytes_per_row = stage_bytes_per_row.min(bytes_per_row) as usize;
            for layer in 0..size.depth_or_array_layers {
                let rows_offset = layer * rows_per_image;
                for row in rows_offset..rows_offset + height_in_blocks {
                    let src_offset = data_layout.offset as u32 + row * bytes_per_row;
                    let dst_offset = row * stage_bytes_per_row;
                    unsafe {
                        staging_buffer.write_with_offset(
                            data,
                            src_offset as isize,
                            dst_offset as isize,
                            copy_bytes_per_row,
                        )
                    }
                }
            }
            staging_buffer
        };

        let staging_buffer = staging_buffer.flush();

        let regions = (0..array_layer_count)
            .map(|array_layer_offset| {
                let mut texture_base = dst_base.clone();
                texture_base.array_layer += array_layer_offset;
                hal::BufferTextureCopy {
                    buffer_layout: wgt::ImageDataLayout {
                        offset: array_layer_offset as u64
                            * rows_per_image as u64
                            * stage_bytes_per_row as u64,
                        bytes_per_row: Some(stage_bytes_per_row),
                        rows_per_image: Some(rows_per_image),
                    },
                    texture_base,
                    size: hal_copy_size,
                }
            })
            .collect::<Vec<_>>();

        {
            let buffer_barrier = hal::BufferBarrier {
                buffer: staging_buffer.raw(),
                usage: hal::BufferUses::MAP_WRITE..hal::BufferUses::COPY_SRC,
            };

            let mut trackers = device.trackers.lock();
            let transition =
                trackers
                    .textures
                    .set_single(&dst, selector, hal::TextureUses::COPY_DST);
            let texture_barriers = transition
                .map(|pending| pending.into_hal(dst_raw))
                .collect::<Vec<_>>();

            unsafe {
                encoder.transition_textures(&texture_barriers);
                encoder.transition_buffers(&[buffer_barrier]);
                encoder.copy_buffer_to_texture(staging_buffer.raw(), dst_raw, &regions);
            }
        }

        pending_writes.consume(staging_buffer);
        pending_writes.insert_texture(&dst);

        Ok(())
    }

    #[cfg(webgl)]
    pub fn queue_copy_external_image_to_texture(
        &self,
        queue_id: QueueId,
        source: &wgt::ImageCopyExternalImage,
        destination: crate::command::ImageCopyTextureTagged,
        size: wgt::Extent3d,
    ) -> Result<(), QueueWriteError> {
        profiling::scope!("Queue::copy_external_image_to_texture");

        let hub = &self.hub;

        let queue = hub.queues.get(queue_id);

        let device = &queue.device;

        if size.width == 0 || size.height == 0 || size.depth_or_array_layers == 0 {
            log::trace!("Ignoring write_texture of size 0");
            return Ok(());
        }

        let mut needs_flag = false;
        needs_flag |= matches!(source.source, wgt::ExternalImageSource::OffscreenCanvas(_));
        needs_flag |= source.origin != wgt::Origin2d::ZERO;
        needs_flag |= destination.color_space != wgt::PredefinedColorSpace::Srgb;
        #[allow(clippy::bool_comparison)]
        if matches!(source.source, wgt::ExternalImageSource::ImageBitmap(_)) {
            needs_flag |= source.flip_y != false;
            needs_flag |= destination.premultiplied_alpha != false;
        }

        if needs_flag {
            device
                .require_downlevel_flags(wgt::DownlevelFlags::UNRESTRICTED_EXTERNAL_TEXTURE_COPIES)
                .map_err(TransferError::from)?;
        }

        let src_width = source.source.width();
        let src_height = source.source.height();

        let dst = hub.textures.get(destination.texture).get()?;

        if !conv::is_valid_external_image_copy_dst_texture_format(dst.desc.format) {
            return Err(
                TransferError::ExternalCopyToForbiddenTextureFormat(dst.desc.format).into(),
            );
        }
        if dst.desc.dimension != wgt::TextureDimension::D2 {
            return Err(TransferError::InvalidDimensionExternal(destination.texture).into());
        }
        dst.check_usage(wgt::TextureUsages::COPY_DST)
            .map_err(TransferError::MissingTextureUsage)?;
        if !dst
            .desc
            .usage
            .contains(wgt::TextureUsages::RENDER_ATTACHMENT)
        {
            return Err(
                TransferError::MissingRenderAttachmentUsageFlag(destination.texture).into(),
            );
        }
        if dst.desc.sample_count != 1 {
            return Err(TransferError::InvalidSampleCount {
                sample_count: dst.desc.sample_count,
            }
            .into());
        }

        if source.origin.x + size.width > src_width {
            return Err(TransferError::TextureOverrun {
                start_offset: source.origin.x,
                end_offset: source.origin.x + size.width,
                texture_size: src_width,
                dimension: crate::resource::TextureErrorDimension::X,
                side: CopySide::Source,
            }
            .into());
        }
        if source.origin.y + size.height > src_height {
            return Err(TransferError::TextureOverrun {
                start_offset: source.origin.y,
                end_offset: source.origin.y + size.height,
                texture_size: src_height,
                dimension: crate::resource::TextureErrorDimension::Y,
                side: CopySide::Source,
            }
            .into());
        }
        if size.depth_or_array_layers != 1 {
            return Err(TransferError::TextureOverrun {
                start_offset: 0,
                end_offset: size.depth_or_array_layers,
                texture_size: 1,
                dimension: crate::resource::TextureErrorDimension::Z,
                side: CopySide::Source,
            }
            .into());
        }

        // Note: Doing the copy range validation early is important because ensures that the
        // dimensions are not going to cause overflow in other parts of the validation.
        let (hal_copy_size, _) = validate_texture_copy_range(
            &destination.to_untagged(),
            &dst.desc,
            CopySide::Destination,
            &size,
        )?;

        let (selector, dst_base) =
            extract_texture_selector(&destination.to_untagged(), &size, &dst)?;

        let mut pending_writes = device.pending_writes.lock();
        let encoder = pending_writes.activate();

        // If the copy does not fully cover the layers, we need to initialize to
        // zero *first* as we don't keep track of partial texture layer inits.
        //
        // Strictly speaking we only need to clear the areas of a layer
        // untouched, but this would get increasingly messy.
        let init_layer_range = if dst.desc.dimension == wgt::TextureDimension::D3 {
            // volume textures don't have a layer range as array volumes aren't supported
            0..1
        } else {
            destination.origin.z..destination.origin.z + size.depth_or_array_layers
        };
        let mut dst_initialization_status = dst.initialization_status.write();
        if dst_initialization_status.mips[destination.mip_level as usize]
            .check(init_layer_range.clone())
            .is_some()
        {
            if has_copy_partial_init_tracker_coverage(&size, destination.mip_level, &dst.desc) {
                for layer_range in dst_initialization_status.mips[destination.mip_level as usize]
                    .drain(init_layer_range)
                    .collect::<Vec<std::ops::Range<u32>>>()
                {
                    let mut trackers = device.trackers.lock();
                    crate::command::clear_texture(
                        &dst,
                        TextureInitRange {
                            mip_range: destination.mip_level..(destination.mip_level + 1),
                            layer_range,
                        },
                        encoder,
                        &mut trackers.textures,
                        &device.alignments,
                        device.zero_buffer.as_ref(),
                        &device.snatchable_lock.read(),
                    )
                    .map_err(QueueWriteError::from)?;
                }
            } else {
                dst_initialization_status.mips[destination.mip_level as usize]
                    .drain(init_layer_range);
            }
        }

        let snatch_guard = device.snatchable_lock.read();
        let dst_raw = dst.try_raw(&snatch_guard)?;

        let regions = hal::TextureCopy {
            src_base: hal::TextureCopyBase {
                mip_level: 0,
                array_layer: 0,
                origin: source.origin.to_3d(0),
                aspect: hal::FormatAspects::COLOR,
            },
            dst_base,
            size: hal_copy_size,
        };

        let mut trackers = device.trackers.lock();
        let transitions = trackers
            .textures
            .set_single(&dst, selector, hal::TextureUses::COPY_DST);

        // `copy_external_image_to_texture` is exclusive to the WebGL backend.
        // Don't go through the `DynCommandEncoder` abstraction and directly to the WebGL backend.
        let encoder_webgl = encoder
            .as_any_mut()
            .downcast_mut::<hal::gles::CommandEncoder>()
            .unwrap();
        let dst_raw_webgl = dst_raw
            .as_any()
            .downcast_ref::<hal::gles::Texture>()
            .unwrap();
        let transitions_webgl = transitions.map(|pending| {
            let dyn_transition = pending.into_hal(dst_raw);
            hal::TextureBarrier {
                texture: dst_raw_webgl,
                range: dyn_transition.range,
                usage: dyn_transition.usage,
            }
        });

        use hal::CommandEncoder as _;
        unsafe {
            encoder_webgl.transition_textures(transitions_webgl);
            encoder_webgl.copy_external_image_to_texture(
                source,
                dst_raw_webgl,
                destination.premultiplied_alpha,
                iter::once(regions),
            );
        }

        Ok(())
    }

    pub fn queue_submit(
        &self,
        queue_id: QueueId,
        command_buffer_ids: &[id::CommandBufferId],
    ) -> Result<SubmissionIndex, (SubmissionIndex, QueueSubmitError)> {
        profiling::scope!("Queue::submit");
        api_log!("Queue::submit {queue_id:?}");

        let submit_index;

        let res = 'error: {
            let hub = &self.hub;

            let queue = hub.queues.get(queue_id);

            let device = &queue.device;

            let snatch_guard = device.snatchable_lock.read();

            // Fence lock must be acquired after the snatch lock everywhere to avoid deadlocks.
            let mut fence = device.fence.write();
            submit_index = device
                .active_submission_index
                .fetch_add(1, Ordering::SeqCst)
                + 1;
            let mut active_executions = Vec::new();

            let mut used_surface_textures = track::TextureUsageScope::default();

            // Use a hashmap here to deduplicate the surface textures that are used in the command buffers.
            // This avoids vulkan deadlocking from the same surface texture being submitted multiple times.
            let mut submit_surface_textures_owned = FastHashMap::default();

            {
                let command_buffer_guard = hub.command_buffers.read();

                if !command_buffer_ids.is_empty() {
                    profiling::scope!("prepare");

                    let mut first_error = None;

                    //TODO: if multiple command buffers are submitted, we can re-use the last
                    // native command buffer of the previous chain instead of always creating
                    // a temporary one, since the chains are not finished.

                    // finish all the command buffers first
                    for command_buffer_id in command_buffer_ids {
                        profiling::scope!("process command buffer");

                        // we reset the used surface textures every time we use
                        // it, so make sure to set_size on it.
                        used_surface_textures.set_size(device.tracker_indices.textures.size());

                        let command_buffer = command_buffer_guard.get(*command_buffer_id);

                        // Note that we are required to invalidate all command buffers in both the success and failure paths.
                        // This is why we `continue` and don't early return via `?`.
                        #[allow(unused_mut)]
                        let mut cmd_buf_data = command_buffer.try_take();

                        #[cfg(feature = "trace")]
                        if let Some(ref mut trace) = *device.trace.lock() {
                            if let Ok(ref mut cmd_buf_data) = cmd_buf_data {
                                trace.add(Action::Submit(
                                    submit_index,
                                    cmd_buf_data.commands.take().unwrap(),
                                ));
                            }
                        }

                        let mut baked = match cmd_buf_data {
                            Ok(cmd_buf_data) => {
                                let res = validate_command_buffer(
                                    &command_buffer,
                                    &queue,
                                    &cmd_buf_data,
                                    &snatch_guard,
                                    &mut submit_surface_textures_owned,
                                    &mut used_surface_textures,
                                );
                                if let Err(err) = res {
                                    first_error.get_or_insert(err);
                                    cmd_buf_data.destroy(&command_buffer.device);
                                    continue;
                                }
                                cmd_buf_data.into_baked_commands()
                            }
                            Err(err) => {
                                first_error.get_or_insert(err.into());
                                continue;
                            }
                        };

                        if first_error.is_some() {
                            continue;
                        }

                        // execute resource transitions
                        if let Err(e) = unsafe {
                            baked.encoder.begin_encoding(hal_label(
                                Some("(wgpu internal) Transit"),
                                device.instance_flags,
                            ))
                        }
                        .map_err(|e| device.handle_hal_error(e))
                        {
                            break 'error Err(e.into());
                        }

                        //Note: locking the trackers has to be done after the storages
                        let mut trackers = device.trackers.lock();
                        if let Err(e) = baked.initialize_buffer_memory(&mut trackers, &snatch_guard)
                        {
                            break 'error Err(e.into());
                        }
                        if let Err(e) =
                            baked.initialize_texture_memory(&mut trackers, device, &snatch_guard)
                        {
                            break 'error Err(e.into());
                        }

                        //Note: stateless trackers are not merged:
                        // device already knows these resources exist.
                        CommandBuffer::insert_barriers_from_device_tracker(
                            baked.encoder.as_mut(),
                            &mut trackers,
                            &baked.trackers,
                            &snatch_guard,
                        );

                        let transit = unsafe { baked.encoder.end_encoding().unwrap() };
                        baked.list.insert(0, transit);

                        // Transition surface textures into `Present` state.
                        // Note: we could technically do it after all of the command buffers,
                        // but here we have a command encoder by hand, so it's easier to use it.
                        if !used_surface_textures.is_empty() {
                            if let Err(e) = unsafe {
                                baked.encoder.begin_encoding(hal_label(
                                    Some("(wgpu internal) Present"),
                                    device.instance_flags,
                                ))
                            }
                            .map_err(|e| device.handle_hal_error(e))
                            {
                                break 'error Err(e.into());
                            }
                            let texture_barriers = trackers
                                .textures
                                .set_from_usage_scope_and_drain_transitions(
                                    &used_surface_textures,
                                    &snatch_guard,
                                )
                                .collect::<Vec<_>>();
                            let present = unsafe {
                                baked.encoder.transition_textures(&texture_barriers);
                                baked.encoder.end_encoding().unwrap()
                            };
                            baked.list.push(present);
                            used_surface_textures = track::TextureUsageScope::default();
                        }

                        // done
                        active_executions.push(EncoderInFlight {
                            raw: baked.encoder,
                            cmd_buffers: baked.list,
                            trackers: baked.trackers,
                            pending_buffers: FastHashMap::default(),
                            pending_textures: FastHashMap::default(),
                        });
                    }

                    if let Some(first_error) = first_error {
                        break 'error Err(first_error);
                    }
                }
            }

            let mut pending_writes = device.pending_writes.lock();

            {
                used_surface_textures.set_size(hub.textures.read().len());
                for texture in pending_writes.dst_textures.values() {
                    match texture.try_inner(&snatch_guard) {
                        Ok(TextureInner::Native { .. }) => {}
                        Ok(TextureInner::Surface { .. }) => {
                            // Compare the Arcs by pointer as Textures don't implement Eq
                            submit_surface_textures_owned
                                .insert(Arc::as_ptr(texture), texture.clone());

                            unsafe {
                                used_surface_textures
                                    .merge_single(texture, None, hal::TextureUses::PRESENT)
                                    .unwrap()
                            };
                        }
                        Err(e) => break 'error Err(e.into()),
                    }
                }

                if !used_surface_textures.is_empty() {
                    let mut trackers = device.trackers.lock();

                    let texture_barriers = trackers
                        .textures
                        .set_from_usage_scope_and_drain_transitions(
                            &used_surface_textures,
                            &snatch_guard,
                        )
                        .collect::<Vec<_>>();
                    unsafe {
                        pending_writes
                            .command_encoder
                            .transition_textures(&texture_barriers);
                    };
                }
            }

            match pending_writes.pre_submit(&device.command_allocator, device, &queue) {
                Ok(Some(pending_execution)) => {
                    active_executions.insert(0, pending_execution);
                }
                Ok(None) => {}
                Err(e) => break 'error Err(e.into()),
            }

            let hal_command_buffers = active_executions
                .iter()
                .flat_map(|e| e.cmd_buffers.iter().map(|b| b.as_ref()))
                .collect::<Vec<_>>();

            {
                let mut submit_surface_textures =
                    SmallVec::<[&dyn hal::DynSurfaceTexture; 2]>::with_capacity(
                        submit_surface_textures_owned.len(),
                    );

                for texture in submit_surface_textures_owned.values() {
                    let raw = match texture.inner.get(&snatch_guard) {
                        Some(TextureInner::Surface { raw, .. }) => raw.as_ref(),
                        _ => unreachable!(),
                    };
                    submit_surface_textures.push(raw);
                }

                if let Err(e) = unsafe {
                    queue.raw().submit(
                        &hal_command_buffers,
                        &submit_surface_textures,
                        (fence.as_mut(), submit_index),
                    )
                }
                .map_err(|e| device.handle_hal_error(e))
                {
                    break 'error Err(e.into());
                }

                // Advance the successful submission index.
                device
                    .last_successful_submission_index
                    .fetch_max(submit_index, Ordering::SeqCst);
            }

            profiling::scope!("cleanup");

            // this will register the new submission to the life time tracker
            device.lock_life().track_submission(
                submit_index,
                pending_writes.temp_resources.drain(..),
                active_executions,
            );
            drop(pending_writes);

            // This will schedule destruction of all resources that are no longer needed
            // by the user but used in the command stream, among other things.
            let fence_guard = RwLockWriteGuard::downgrade(fence);
            let (closures, _) =
                match device.maintain(fence_guard, wgt::Maintain::Poll, snatch_guard) {
                    Ok(closures) => closures,
                    Err(WaitIdleError::Device(err)) => {
                        break 'error Err(QueueSubmitError::Queue(err))
                    }
                    Err(WaitIdleError::StuckGpu) => break 'error Err(QueueSubmitError::StuckGpu),
                    Err(WaitIdleError::WrongSubmissionIndex(..)) => unreachable!(),
                };

            Ok(closures)
        };

        let callbacks = match res {
            Ok(ok) => ok,
            Err(e) => return Err((submit_index, e)),
        };

        // the closures should execute with nothing locked!
        callbacks.fire();

        api_log!("Queue::submit to {queue_id:?} returned submit index {submit_index}");

        Ok(submit_index)
    }

    pub fn queue_get_timestamp_period(&self, queue_id: QueueId) -> f32 {
        let queue = self.hub.queues.get(queue_id);
        unsafe { queue.raw().get_timestamp_period() }
    }

    pub fn queue_on_submitted_work_done(
        &self,
        queue_id: QueueId,
        closure: SubmittedWorkDoneClosure,
    ) {
        api_log!("Queue::on_submitted_work_done {queue_id:?}");

        //TODO: flush pending writes
        let queue = self.hub.queues.get(queue_id);
        queue.device.lock_life().add_work_done_closure(closure);
    }
}

fn validate_command_buffer(
    command_buffer: &CommandBuffer,
    queue: &Queue,
    cmd_buf_data: &crate::command::CommandBufferMutable,
    snatch_guard: &crate::snatch::SnatchGuard<'_>,
    submit_surface_textures_owned: &mut FastHashMap<*const Texture, Arc<Texture>>,
    used_surface_textures: &mut track::TextureUsageScope,
) -> Result<(), QueueSubmitError> {
    command_buffer.same_device_as(queue)?;
    cmd_buf_data.check_finished()?;

    {
        profiling::scope!("check resource state");

        {
            profiling::scope!("buffers");
            for buffer in cmd_buf_data.trackers.buffers.used_resources() {
                buffer.check_destroyed(snatch_guard)?;

                match *buffer.map_state.lock() {
                    BufferMapState::Idle => (),
                    _ => return Err(QueueSubmitError::BufferStillMapped(buffer.error_ident())),
                }
            }
        }
        {
            profiling::scope!("textures");
            for texture in cmd_buf_data.trackers.textures.used_resources() {
                let should_extend = match texture.try_inner(snatch_guard)? {
                    TextureInner::Native { .. } => false,
                    TextureInner::Surface { .. } => {
                        // Compare the Arcs by pointer as Textures don't implement Eq.
                        submit_surface_textures_owned
                            .insert(Arc::as_ptr(&texture), texture.clone());

                        true
                    }
                };
                if should_extend {
                    unsafe {
                        used_surface_textures
                            .merge_single(&texture, None, hal::TextureUses::PRESENT)
                            .unwrap();
                    };
                }
            }
        }
    }
    Ok(())
}