/* * struct rcar_dmac_hw_desc - Hardware descriptor for a transfer chunk * @sar: value of the SAR register (source address) * @dar: value of the DAR register (destination address) * @tcr: value of the TCR register (transfer count)
*/ struct rcar_dmac_hw_desc {
u32 sar;
u32 dar;
u32 tcr;
u32 reserved;
} __attribute__((__packed__));
/* * struct rcar_dmac_desc - R-Car Gen2 DMA Transfer Descriptor * @async_tx: base DMA asynchronous transaction descriptor * @direction: direction of the DMA transfer * @xfer_shift: log2 of the transfer size * @chcr: value of the channel configuration register for this transfer * @node: entry in the channel's descriptors lists * @chunks: list of transfer chunks for this transfer * @running: the transfer chunk being currently processed * @nchunks: number of transfer chunks for this transfer * @hwdescs.use: whether the transfer descriptor uses hardware descriptors * @hwdescs.mem: hardware descriptors memory for the transfer * @hwdescs.dma: device address of the hardware descriptors memory * @hwdescs.size: size of the hardware descriptors in bytes * @size: transfer size in bytes * @cyclic: when set indicates that the DMA transfer is cyclic
*/ struct rcar_dmac_desc { struct dma_async_tx_descriptor async_tx; enum dma_transfer_direction direction; unsignedint xfer_shift;
u32 chcr;
/* * struct rcar_dmac_desc_page - One page worth of descriptors * @node: entry in the channel's pages list * @descs: array of DMA descriptors * @chunks: array of transfer chunk descriptors
*/ struct rcar_dmac_desc_page { struct list_head node;
union {
DECLARE_FLEX_ARRAY(struct rcar_dmac_desc, descs);
DECLARE_FLEX_ARRAY(struct rcar_dmac_xfer_chunk, chunks);
};
};
/* * struct rcar_dmac_chan_map - Map of slave device phys to dma address * @addr: slave dma address * @dir: direction of mapping * @slave: slave configuration that is mapped
*/ struct rcar_dmac_chan_map {
dma_addr_t addr; enum dma_data_direction dir; struct rcar_dmac_chan_slave slave;
};
/* * struct rcar_dmac_chan - R-Car Gen2 DMA Controller Channel * @chan: base DMA channel object * @iomem: channel I/O memory base * @index: index of this channel in the controller * @irq: channel IRQ * @src: slave memory address and size on the source side * @dst: slave memory address and size on the destination side * @mid_rid: hardware MID/RID for the DMA client using this channel * @lock: protects the channel CHCR register and the desc members * @desc.free: list of free descriptors * @desc.pending: list of pending descriptors (submitted with tx_submit) * @desc.active: list of active descriptors (activated with issue_pending) * @desc.done: list of completed descriptors * @desc.wait: list of descriptors waiting for an ack * @desc.running: the descriptor being processed (a member of the active list) * @desc.chunks_free: list of free transfer chunk descriptors * @desc.pages: list of pages used by allocated descriptors
*/ struct rcar_dmac_chan { struct dma_chan chan; void __iomem *iomem; unsignedint index; int irq;
/* * Errata: When descriptor memory is accessed through an IOMMU * the DMADAR register isn't initialized automatically from the * first descriptor at beginning of transfer by the DMAC like it * should. Initialize it manually with the destination address * of the first chunk.
*/
rcar_dmac_chan_write(chan, RCAR_DMADAR,
chunk->dst_addr & 0xffffffff);
/* * Program the descriptor stage interrupt to occur after the end * of the first stage.
*/
rcar_dmac_chan_write(chan, RCAR_DMADPCR, RCAR_DMADPCR_DIPT(1));
/* * If the descriptor isn't cyclic enable normal descriptor mode * and the transfer completion interrupt.
*/ if (!desc->cyclic)
chcr |= RCAR_DMACHCR_DPM_ENABLED | RCAR_DMACHCR_IE; /* * If the descriptor is cyclic and has a callback enable the * descriptor stage interrupt in infinite repeat mode.
*/ elseif (desc->async_tx.callback)
chcr |= RCAR_DMACHCR_DPM_INFINITE | RCAR_DMACHCR_DSIE; /* * Otherwise just select infinite repeat mode without any * interrupt.
*/ else
chcr |= RCAR_DMACHCR_DPM_INFINITE;
} else { struct rcar_dmac_xfer_chunk *chunk = desc->running;
/* * rcar_dmac_desc_put - Release a DMA transfer descriptor * @chan: the DMA channel * @desc: the descriptor * * Put the descriptor and its transfer chunk descriptors back in the channel's * free descriptors lists. The descriptor's chunks list will be reinitialized to * an empty list as a result. * * The descriptor must have been removed from the channel's lists before calling * this function.
*/ staticvoid rcar_dmac_desc_put(struct rcar_dmac_chan *chan, struct rcar_dmac_desc *desc)
{ unsignedlong flags;
/* * We have to temporarily move all descriptors from the wait list to a * local list as iterating over the wait list, even with * list_for_each_entry_safe, isn't safe if we release the channel lock * around the rcar_dmac_desc_put() call.
*/
spin_lock_irqsave(&chan->lock, flags);
list_splice_init(&chan->desc.wait, &list);
spin_unlock_irqrestore(&chan->lock, flags);
/* Put the remaining descriptors back in the wait list. */
spin_lock_irqsave(&chan->lock, flags);
list_splice(&list, &chan->desc.wait);
spin_unlock_irqrestore(&chan->lock, flags);
}
/* * rcar_dmac_desc_get - Allocate a descriptor for a DMA transfer * @chan: the DMA channel * * Locking: This function must be called in a non-atomic context. * * Return: A pointer to the allocated descriptor or NULL if no descriptor can * be allocated.
*/ staticstruct rcar_dmac_desc *rcar_dmac_desc_get(struct rcar_dmac_chan *chan)
{ struct rcar_dmac_desc *desc; unsignedlong flags; int ret;
/* Recycle acked descriptors before attempting allocation. */
rcar_dmac_desc_recycle_acked(chan);
spin_lock_irqsave(&chan->lock, flags);
while (list_empty(&chan->desc.free)) { /* * No free descriptors, allocate a page worth of them and try * again, as someone else could race us to get the newly * allocated descriptors. If the allocation fails return an * error.
*/
spin_unlock_irqrestore(&chan->lock, flags);
ret = rcar_dmac_desc_alloc(chan, GFP_NOWAIT); if (ret < 0) return NULL;
spin_lock_irqsave(&chan->lock, flags);
}
/* * rcar_dmac_xfer_chunk_get - Allocate a transfer chunk for a DMA transfer * @chan: the DMA channel * * Locking: This function must be called in a non-atomic context. * * Return: A pointer to the allocated transfer chunk descriptor or NULL if no * descriptor can be allocated.
*/ staticstruct rcar_dmac_xfer_chunk *
rcar_dmac_xfer_chunk_get(struct rcar_dmac_chan *chan)
{ struct rcar_dmac_xfer_chunk *chunk; unsignedlong flags; int ret;
spin_lock_irqsave(&chan->lock, flags);
while (list_empty(&chan->desc.chunks_free)) { /* * No free descriptors, allocate a page worth of them and try * again, as someone else could race us to get the newly * allocated descriptors. If the allocation fails return an * error.
*/
spin_unlock_irqrestore(&chan->lock, flags);
ret = rcar_dmac_xfer_chunk_alloc(chan, GFP_NOWAIT); if (ret < 0) return NULL;
spin_lock_irqsave(&chan->lock, flags);
}
staticvoid rcar_dmac_realloc_hwdesc(struct rcar_dmac_chan *chan, struct rcar_dmac_desc *desc, size_t size)
{ /* * dma_alloc_coherent() allocates memory in page size increments. To * avoid reallocating the hardware descriptors when the allocated size * wouldn't change align the requested size to a multiple of the page * size.
*/
size = PAGE_ALIGN(size);
/* * Ensure that the setting of the DE bit is actually 0 after * clearing it.
*/ for (i = 0; i < 1024; i++) {
chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR); if (!(chcr & RCAR_DMACHCR_DE)) return;
udelay(1);
}
dev_err(chan->chan.device->dev, "CHCR DE check error\n");
}
/* Move all non-free descriptors to the local lists. */
list_splice_init(&chan->desc.pending, &descs);
list_splice_init(&chan->desc.active, &descs);
list_splice_init(&chan->desc.done, &descs);
list_splice_init(&chan->desc.wait, &descs);
/* * rcar_dmac_chan_prep_sg - prepare transfer descriptors from an SG list * * Common routine for public (MEMCPY) and slave DMA. The MEMCPY case is also * converted to scatter-gather to guarantee consistent locking and a correct * list manipulation. For slave DMA direction carries the usual meaning, and, * logically, the SG list is RAM and the addr variable contains slave address, * e.g., the FIFO I/O register. For MEMCPY direction equals DMA_MEM_TO_MEM * and the SG list contains only one element and points at the source buffer.
*/ staticstruct dma_async_tx_descriptor *
rcar_dmac_chan_prep_sg(struct rcar_dmac_chan *chan, struct scatterlist *sgl, unsignedint sg_len, dma_addr_t dev_addr, enum dma_transfer_direction dir, unsignedlong dma_flags, bool cyclic)
{ struct rcar_dmac_xfer_chunk *chunk; struct rcar_dmac_desc *desc; struct scatterlist *sg; unsignedint nchunks = 0; unsignedint max_chunk_size; unsignedint full_size = 0; bool cross_boundary = false; unsignedint i; #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
u32 high_dev_addr;
u32 high_mem_addr; #endif
desc = rcar_dmac_desc_get(chan); if (!desc) return NULL;
/* * Allocate and fill the transfer chunk descriptors. We own the only * reference to the DMA descriptor, there's no need for locking.
*/
for_each_sg(sgl, sg, sg_len, i) {
dma_addr_t mem_addr = sg_dma_address(sg); unsignedint len = sg_dma_len(sg);
full_size += len;
#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT if (i == 0) {
high_dev_addr = dev_addr >> 32;
high_mem_addr = mem_addr >> 32;
}
/* * Use hardware descriptor lists if possible when more than one chunk * needs to be transferred (otherwise they don't make much sense). * * Source/Destination address should be located in same 4GiB region * in the 40bit address space when it uses Hardware descriptor, * and cross_boundary is checking it.
*/
desc->hwdescs.use = !cross_boundary && nchunks > 1; if (desc->hwdescs.use) { if (rcar_dmac_fill_hwdesc(chan, desc) < 0)
desc->hwdescs.use = false;
}
/* Protect against ISR */
spin_lock_irq(&rchan->lock);
rcar_dmac_chan_halt(rchan);
spin_unlock_irq(&rchan->lock);
/* * Now no new interrupts will occur, but one might already be * running. Wait for it to finish before freeing resources.
*/
synchronize_irq(rchan->irq);
if (rchan->mid_rid >= 0) { /* The caller is holding dma_list_mutex */
clear_bit(rchan->mid_rid, dmac->modules);
rchan->mid_rid = -EINVAL;
}
/* * Allocate the sg list dynamically as it would consume too much stack * space.
*/
sgl = kmalloc_array(sg_len, sizeof(*sgl), GFP_NOWAIT); if (!sgl) return NULL;
sg_init_table(sgl, sg_len);
for (i = 0; i < sg_len; ++i) {
dma_addr_t src = buf_addr + (period_len * i);
/* * We could lock this, but you shouldn't be configuring the * channel, while using it...
*/
rchan->src.slave_addr = cfg->src_addr;
rchan->dst.slave_addr = cfg->dst_addr;
rchan->src.xfer_size = cfg->src_addr_width;
rchan->dst.xfer_size = cfg->dst_addr_width;
/* * If the cookie corresponds to a descriptor that has been completed * there is no residue. The same check has already been performed by the * caller but without holding the channel lock, so the descriptor could * now be complete.
*/
status = dma_cookie_status(&chan->chan, cookie, NULL); if (status == DMA_COMPLETE) return 0;
/* * If the cookie doesn't correspond to the currently running transfer * then the descriptor hasn't been processed yet, and the residue is * equal to the full descriptor size. * Also, a client driver is possible to call this function before * rcar_dmac_isr_channel_thread() runs. In this case, the "desc.running" * will be the next descriptor, and the done list will appear. So, if * the argument cookie matches the done list's cookie, we can assume * the residue is zero.
*/ if (cookie != desc->async_tx.cookie) {
list_for_each_entry(desc, &chan->desc.done, node) { if (cookie == desc->async_tx.cookie) return 0;
}
list_for_each_entry(desc, &chan->desc.pending, node) { if (cookie == desc->async_tx.cookie) return desc->size;
}
list_for_each_entry(desc, &chan->desc.active, node) { if (cookie == desc->async_tx.cookie) return desc->size;
}
/* * No descriptor found for the cookie, there's thus no residue. * This shouldn't happen if the calling driver passes a correct * cookie value.
*/
WARN(1, "No descriptor for cookie!"); return 0;
}
/* * We need to read two registers. * Make sure the control register does not skip to next chunk * while reading the counter. * Trying it 3 times should be enough: Initial read, retry, retry * for the paranoid.
*/ for (i = 0; i < 3; i++) {
chcrb = rcar_dmac_chan_read(chan, RCAR_DMACHCRB) &
RCAR_DMACHCRB_DPTR_MASK;
tcrb = rcar_dmac_chan_read(chan, RCAR_DMATCRB); /* Still the same? */ if (chcrb == (rcar_dmac_chan_read(chan, RCAR_DMACHCRB) &
RCAR_DMACHCRB_DPTR_MASK)) break;
}
WARN_ONCE(i >= 3, "residue might be not continuous!");
/* * In descriptor mode the descriptor running pointer is not maintained * by the interrupt handler, find the running descriptor from the * descriptor pointer field in the CHCRB register. In non-descriptor * mode just use the running descriptor pointer.
*/ if (desc->hwdescs.use) {
dptr = chcrb >> RCAR_DMACHCRB_DPTR_SHIFT; if (dptr == 0)
dptr = desc->nchunks;
dptr--;
WARN_ON(dptr >= desc->nchunks);
} else {
running = desc->running;
}
/* Compute the size of all chunks still to be transferred. */
list_for_each_entry_reverse(chunk, &desc->chunks, node) { if (chunk == running || ++dptr == desc->nchunks) break;
residue += chunk->size;
}
/* Add the residue for the current chunk. */
residue += tcrb << desc->xfer_shift;
/* Append the pending list to the active list. */
list_splice_tail_init(&rchan->desc.pending, &rchan->desc.active);
/* * If no transfer is running pick the first descriptor from the active * list and start the transfer.
*/ if (!rchan->desc.running) { struct rcar_dmac_desc *desc;
if (WARN_ON(!desc || !desc->cyclic)) { /* * This should never happen, there should always be a running * cyclic descriptor when a descriptor stage end interrupt is * triggered. Warn and return.
*/ return IRQ_NONE;
}
/* Program the interrupt pointer to the next stage. */
stage = (rcar_dmac_chan_read(chan, RCAR_DMACHCRB) &
RCAR_DMACHCRB_DPTR_MASK) >> RCAR_DMACHCRB_DPTR_SHIFT;
rcar_dmac_chan_write(chan, RCAR_DMADPCR, RCAR_DMADPCR_DIPT(stage));
if (WARN_ON_ONCE(!desc)) { /* * This should never happen, there should always be a running * descriptor when a transfer end interrupt is triggered. Warn * and return.
*/ return IRQ_NONE;
}
/* * The transfer end interrupt isn't generated for each chunk when using * descriptor mode. Only update the running chunk pointer in * non-descriptor mode.
*/ if (!desc->hwdescs.use) { /* * If we haven't completed the last transfer chunk simply move * to the next one. Only wake the IRQ thread if the transfer is * cyclic.
*/ if (!list_is_last(&desc->running->node, &desc->chunks)) {
desc->running = list_next_entry(desc->running, node); if (!desc->cyclic)
ret = IRQ_HANDLED; goto done;
}
/* * We've completed the last transfer chunk. If the transfer is * cyclic, move back to the first one.
*/ if (desc->cyclic) {
desc->running =
list_first_entry(&desc->chunks, struct rcar_dmac_xfer_chunk,
node); goto done;
}
}
/* The descriptor is complete, move it to the done list. */
list_move_tail(&desc->node, &chan->desc.done);
/* Queue the next descriptor, if any. */ if (!list_empty(&chan->desc.active))
chan->desc.running = list_first_entry(&chan->desc.active, struct rcar_dmac_desc,
node); else
chan->desc.running = NULL;
done: if (chan->desc.running)
rcar_dmac_chan_start_xfer(chan);
/* * We don't need to call rcar_dmac_chan_halt() * because channel is already stopped in error case. * We need to clear register and check DE bit as recovery.
*/
rcar_dmac_chan_clear(dmac, chan);
rcar_dmac_chcr_de_barrier(chan);
reinit = true; goto spin_lock_end;
}
if (chcr & RCAR_DMACHCR_TE)
mask |= RCAR_DMACHCR_DE;
rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr & ~mask); if (mask & RCAR_DMACHCR_DE)
rcar_dmac_chcr_de_barrier(chan);
if (chcr & RCAR_DMACHCR_DSE)
ret |= rcar_dmac_isr_desc_stage_end(chan);
if (chcr & RCAR_DMACHCR_TE)
ret |= rcar_dmac_isr_transfer_end(chan);
spin_lock_end:
spin_unlock(&chan->lock);
if (reinit) {
dev_err(chan->chan.device->dev, "Channel Address Error\n");
/* For cyclic transfers notify the user after every chunk. */ if (chan->desc.running && chan->desc.running->cyclic) {
desc = chan->desc.running;
dmaengine_desc_get_callback(&desc->async_tx, &cb);
if (dmaengine_desc_callback_valid(&cb)) {
spin_unlock_irq(&chan->lock);
dmaengine_desc_callback_invoke(&cb, NULL);
spin_lock_irq(&chan->lock);
}
}
/* * Call the callback function for all descriptors on the done list and * move them to the ack wait list.
*/ while (!list_empty(&chan->desc.done)) {
desc = list_first_entry(&chan->desc.done, struct rcar_dmac_desc,
node);
dma_cookie_complete(&desc->async_tx);
list_del(&desc->node);
dmaengine_desc_get_callback(&desc->async_tx, &cb); if (dmaengine_desc_callback_valid(&cb)) {
spin_unlock_irq(&chan->lock); /* * We own the only reference to this descriptor, we can * safely dereference it without holding the channel * lock.
*/
dmaengine_desc_callback_invoke(&cb, NULL);
spin_lock_irq(&chan->lock);
}
list_add_tail(&desc->node, &chan->desc.wait);
}
spin_unlock_irq(&chan->lock);
/* Recycle all acked descriptors. */
rcar_dmac_desc_recycle_acked(chan);
return IRQ_HANDLED;
}
/* ----------------------------------------------------------------------------- * OF xlate and channel filter
*/
/* * FIXME: Using a filter on OF platforms is a nonsense. The OF xlate * function knows from which device it wants to allocate a channel from, * and would be perfectly capable of selecting the channel it wants. * Forcing it to call dma_request_channel() and iterate through all * channels from all controllers is just pointless.
*/ if (chan->device->device_config != rcar_dmac_device_config) returnfalse;
staticconststruct dev_pm_ops rcar_dmac_pm = { /* * TODO for system sleep/resume: * - Wait for the current transfer to complete and stop the device, * - Resume transfers, if any.
*/
SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
pm_runtime_force_resume)
SET_RUNTIME_PM_OPS(rcar_dmac_runtime_suspend, rcar_dmac_runtime_resume,
NULL)
};
/* ----------------------------------------------------------------------------- * Probe and remove
*/
ret = of_property_read_u32(np, "dma-channels", &dmac->n_channels); if (ret < 0) {
dev_err(dev, "unable to read dma-channels property\n"); return ret;
}
/* The hardware and driver don't support more than 32 bits in CHCLR */ if (dmac->n_channels <= 0 ||
dmac->n_channels >= RCAR_DMAC_MAX_CHANNELS) {
dev_err(dev, "invalid number of channels %u\n",
dmac->n_channels); return -EINVAL;
}
/* * If the driver is unable to read dma-channel-mask property, * the driver assumes that it can use all channels.
*/
dmac->channels_mask = GENMASK(dmac->n_channels - 1, 0);
of_property_read_u32(np, "dma-channel-mask", &dmac->channels_mask);
/* If the property has out-of-channel mask, this driver clears it */
dmac->channels_mask &= GENMASK(dmac->n_channels - 1, 0);
ret = dma_set_mask_and_coherent(dmac->dev, DMA_BIT_MASK(40)); if (ret) return ret;
ret = rcar_dmac_parse_of(&pdev->dev, dmac); if (ret < 0) return ret;
/* * A still unconfirmed hardware bug prevents the IPMMU microTLB 0 to be * flushed correctly, resulting in memory corruption. DMAC 0 channel 0 * is connected to microTLB 0 on currently supported platforms, so we * can't use it with the IPMMU. As the IOMMU API operates at the device * level we can't disable it selectively, so ignore channel 0 for now if * the device is part of an IOMMU group.
*/ if (device_iommu_mapped(&pdev->dev))
dmac->channels_mask &= ~BIT(0);
dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels, sizeof(*dmac->channels), GFP_KERNEL); if (!dmac->channels) return -ENOMEM;
for_each_rcar_dmac_chan(i, dmac, chan) {
ret = rcar_dmac_chan_probe(dmac, chan); if (ret < 0) goto err_pm_disable;
}
/* Register the DMAC as a DMA provider for DT. */
ret = of_dma_controller_register(pdev->dev.of_node, rcar_dmac_of_xlate,
NULL); if (ret < 0) goto err_pm_disable;
/* * Register the DMA engine device. * * Default transfer size of 32 bytes requires 32-byte alignment.
*/
ret = dma_async_device_register(engine); if (ret < 0) goto err_dma_free;
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.
