/*************** DMA-QUEUE-GENERAL-FUNCTIONS ***** * DMA services * * Theory of operation * * A Tx or Rx queue resides in host DRAM, and is comprised of a circular buffer * of buffer descriptors, each of which points to one or more data buffers for * the device to read from or fill. Driver and device exchange status of each * queue via "read" and "write" pointers. Driver keeps minimum of 2 empty * entries in each circular buffer, to protect against confusing empty and full * queue states. * * The device reads or writes the data in the queues via the device's several * DMA/FIFO channels. Each queue is mapped to a single DMA channel. * * For Tx queue, there are low mark and high mark limits. If, after queuing * the packet for Tx, free space become < low mark, Tx queue stopped. When * reclaiming packets (on 'tx done IRQ), if free space become > high mark, * Tx queue resumed. *
***************************************************/
int iwl_pcie_alloc_dma_ptr(struct iwl_trans *trans, struct iwl_dma_ptr *ptr, size_t size)
{ if (WARN_ON(ptr->addr)) return -EINVAL;
/* * iwl_pcie_txq_inc_wr_ptr - Send new write index to hardware
*/ staticvoid iwl_pcie_txq_inc_wr_ptr(struct iwl_trans *trans, struct iwl_txq *txq)
{
u32 reg = 0; int txq_id = txq->id;
lockdep_assert_held(&txq->lock);
/* * explicitly wake up the NIC if: * 1. shadow registers aren't enabled * 2. NIC is woken up for CMD regardless of shadow outside this function * 3. there is a chance that the NIC is asleep
*/ if (!trans->mac_cfg->base->shadow_reg_enable &&
txq_id != trans->conf.cmd_queue &&
test_bit(STATUS_TPOWER_PMI, &trans->status)) { /* * wake up nic if it's powered down ... * uCode will wake up, and interrupt us again, so next * time we'll skip this part.
*/
reg = iwl_read32(trans, CSR_UCODE_DRV_GP1);
/* * if not in power-save mode, uCode will never sleep when we're * trying to tx (during RFKILL, we're not trying to tx).
*/
IWL_DEBUG_TX(trans, "Q:%d WR: 0x%x\n", txq_id, txq->write_ptr); if (!txq->block)
iwl_write32(trans, HBUS_TARG_WRPTR,
txq->write_ptr | (txq_id << 8));
}
if (reset)
memset(tfd, 0, trans_pcie->txqs.tfd.size);
num_tbs = iwl_txq_gen1_tfd_get_num_tbs(tfd);
/* Each TFD can point to a maximum max_tbs Tx buffers */ if (num_tbs >= trans_pcie->txqs.tfd.max_tbs) {
IWL_ERR(trans, "Error can not send more than %d chunks\n",
trans_pcie->txqs.tfd.max_tbs); return -EINVAL;
}
/* Decrease internal use count and unmap/free page if needed */ if (refcount_dec_and_test(&info->use_count)) {
dma_unmap_page(trans->dev, info->dma_addr, PAGE_SIZE,
DMA_TO_DEVICE);
if (sizeof(dma_addr_t) <= sizeof(u32)) return addr;
hi_len = le16_to_cpu(tb->hi_n_len) & 0xF;
/* * shift by 16 twice to avoid warnings on 32-bit * (where this code never runs anyway due to the * if statement above)
*/ return addr | ((hi_len << 16) << 16);
}
staticvoid iwl_txq_gen1_tfd_unmap(struct iwl_trans *trans, struct iwl_cmd_meta *meta, struct iwl_txq *txq, int index)
{ struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); int i, num_tbs; struct iwl_tfd *tfd = iwl_txq_get_tfd(trans, txq, index);
/* Sanity check on number of chunks */
num_tbs = iwl_txq_gen1_tfd_get_num_tbs(tfd);
if (num_tbs > trans_pcie->txqs.tfd.max_tbs) {
IWL_ERR(trans, "Too many chunks: %i\n", num_tbs); /* @todo issue fatal error, it is quite serious situation */ return;
}
/* TB1 is mapped directly, the rest is the TSO page and SG list. */ if (meta->sg_offset)
num_tbs = 2;
/* first TB is never freed - it's the bidirectional DMA data */
for (i = 1; i < num_tbs; i++) { if (meta->tbs & BIT(i))
dma_unmap_page(trans->dev,
iwl_txq_gen1_tfd_tb_get_addr(tfd, i),
iwl_txq_gen1_tfd_tb_get_len(trans,
tfd, i),
DMA_TO_DEVICE); else
dma_unmap_single(trans->dev,
iwl_txq_gen1_tfd_tb_get_addr(tfd, i),
iwl_txq_gen1_tfd_tb_get_len(trans,
tfd, i),
DMA_TO_DEVICE);
}
meta->tbs = 0;
iwl_txq_set_tfd_invalid_gen1(trans, tfd);
}
/** * iwl_txq_free_tfd - Free all chunks referenced by TFD [txq->q.read_ptr] * @trans: transport private data * @txq: tx queue * @read_ptr: the TXQ read_ptr to free * * Does NOT advance any TFD circular buffer read/write indexes * Does NOT free the TFD itself (which is within circular buffer)
*/ staticvoid iwl_txq_free_tfd(struct iwl_trans *trans, struct iwl_txq *txq, int read_ptr)
{ /* rd_ptr is bounded by TFD_QUEUE_SIZE_MAX and * idx is bounded by n_window
*/ int idx = iwl_txq_get_cmd_index(txq, read_ptr); struct sk_buff *skb;
lockdep_assert_held(&txq->reclaim_lock);
if (!txq->entries) return;
/* We have only q->n_window txq->entries, but we use * TFD_QUEUE_SIZE_MAX tfds
*/ if (trans->mac_cfg->gen2)
iwl_txq_gen2_tfd_unmap(trans, &txq->entries[idx].meta,
iwl_txq_get_tfd(trans, txq, read_ptr)); else
iwl_txq_gen1_tfd_unmap(trans, &txq->entries[idx].meta,
txq, read_ptr);
/* free SKB */
skb = txq->entries[idx].skb;
/* Can be called from irqs-disabled context * If skb is not NULL, it means that the whole queue is being * freed and that the queue is not empty - free the skb
*/ if (skb) {
iwl_op_mode_free_skb(trans->op_mode, skb);
txq->entries[idx].skb = NULL;
}
}
/* * iwl_pcie_txq_unmap - Unmap any remaining DMA mappings and free skb's
*/ staticvoid iwl_pcie_txq_unmap(struct iwl_trans *trans, int txq_id)
{ struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_txq *txq = trans_pcie->txqs.txq[txq_id];
if (!txq) {
IWL_ERR(trans, "Trying to free a queue that wasn't allocated?\n"); return;
}
/* just in case - this queue may have been stopped */
iwl_trans_pcie_wake_queue(trans, txq);
}
/* * iwl_pcie_txq_free - Deallocate DMA queue. * @txq: Transmit queue to deallocate. * * Empty queue by removing and destroying all BD's. * Free all buffers. * 0-fill, but do not free "txq" descriptor structure.
*/ staticvoid iwl_pcie_txq_free(struct iwl_trans *trans, int txq_id)
{ struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_txq *txq = trans_pcie->txqs.txq[txq_id]; struct device *dev = trans->dev; int i;
if (WARN_ON(!txq)) return;
iwl_pcie_txq_unmap(trans, txq_id);
/* De-alloc array of command/tx buffers */ if (txq_id == trans->conf.cmd_queue) for (i = 0; i < txq->n_window; i++) {
kfree_sensitive(txq->entries[i].cmd);
kfree_sensitive(txq->entries[i].free_buf);
}
void iwl_pcie_tx_start(struct iwl_trans *trans)
{ struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); int nq = trans->mac_cfg->base->num_of_queues; int chan;
u32 reg_val; int clear_dwords = (SCD_TRANS_TBL_OFFSET_QUEUE(nq) -
SCD_CONTEXT_MEM_LOWER_BOUND) / sizeof(u32);
/* make sure all queue are not stopped/used */
memset(trans_pcie->txqs.queue_stopped, 0, sizeof(trans_pcie->txqs.queue_stopped));
memset(trans_pcie->txqs.queue_used, 0, sizeof(trans_pcie->txqs.queue_used));
/* The chain extension of the SCD doesn't work well. This feature is * enabled by default by the HW, so we need to disable it manually.
*/ if (trans->mac_cfg->base->scd_chain_ext_wa)
iwl_write_prph(trans, SCD_CHAINEXT_EN, 0);
/* Tell NIC where to find the "keep warm" buffer */
iwl_write_direct32(trans, FH_KW_MEM_ADDR_REG,
trans_pcie->kw.dma >> 4);
/* * Send 0 as the scd_base_addr since the device may have be reset * while we were in WoWLAN in which case SCD_SRAM_BASE_ADDR will * contain garbage.
*/
iwl_pcie_tx_start(trans);
}
/* Wait for DMA channels to be idle */
ret = iwl_poll_bits(trans, FH_TSSR_TX_STATUS_REG, mask, 5000); if (ret)
IWL_ERR(trans, "Failing on timeout while stopping DMA channel %d [0x%08x]\n",
ch, iwl_read32(trans, FH_TSSR_TX_STATUS_REG));
iwl_trans_release_nic_access(trans);
out:
spin_unlock_bh(&trans_pcie->irq_lock);
}
/* * iwl_pcie_tx_stop - Stop all Tx DMA channels
*/ int iwl_pcie_tx_stop(struct iwl_trans *trans)
{ struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); int txq_id;
/* Turn off all Tx DMA fifos */
iwl_scd_deactivate_fifos(trans);
/* Turn off all Tx DMA channels */
iwl_pcie_tx_stop_fh(trans);
/* * This function can be called before the op_mode disabled the * queues. This happens when we have an rfkill interrupt. * Since we stop Tx altogether - mark the queues as stopped.
*/
memset(trans_pcie->txqs.queue_stopped, 0, sizeof(trans_pcie->txqs.queue_stopped));
memset(trans_pcie->txqs.queue_used, 0, sizeof(trans_pcie->txqs.queue_used));
/* This can happen: start_hw, stop_device */ if (!trans_pcie->txq_memory) return 0;
/* Unmap DMA from host system and free skb's */ for (txq_id = 0; txq_id < trans->mac_cfg->base->num_of_queues;
txq_id++)
iwl_pcie_txq_unmap(trans, txq_id);
return 0;
}
/* * iwl_trans_tx_free - Free TXQ Context * * Destroy all TX DMA queues and structures
*/ void iwl_pcie_tx_free(struct iwl_trans *trans)
{ int txq_id; struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
if (trans->mac_cfg->gen2) {
IWL_ERR(trans, "Queue %d is stuck %d %d\n", txq_id,
txq->read_ptr, txq->write_ptr); /* TODO: access new SCD registers and dump them */ return;
}
status = iwl_read_prph(trans, SCD_QUEUE_STATUS_BITS(txq_id));
fifo = (status >> SCD_QUEUE_STTS_REG_POS_TXF) & 0x7;
active = !!(status & BIT(SCD_QUEUE_STTS_REG_POS_ACTIVE));
IWL_ERR(trans, "Queue %d is %sactive on fifo %d and stuck for %u ms. SW [%d, %d] HW [%d, %d] FH TRB=0x0%x\n",
txq_id, active ? "" : "in", fifo,
jiffies_to_msecs(txq->wd_timeout),
txq->read_ptr, txq->write_ptr,
iwl_read_prph(trans, SCD_QUEUE_RDPTR(txq_id)) &
(trans->mac_cfg->base->max_tfd_queue_size - 1),
iwl_read_prph(trans, SCD_QUEUE_WRPTR(txq_id)) &
(trans->mac_cfg->base->max_tfd_queue_size - 1),
iwl_read_direct32(trans, FH_TX_TRB_REG(fifo)));
}
if (cmd_queue) for (i = 0; i < slots_num; i++) {
txq->entries[i].cmd =
kmalloc(sizeof(struct iwl_device_cmd),
GFP_KERNEL); if (!txq->entries[i].cmd) goto error;
}
/* Circular buffer of transmit frame descriptors (TFDs), * shared with device
*/
txq->tfds = dma_alloc_coherent(trans->dev, tfd_sz,
&txq->dma_addr, GFP_KERNEL); if (!txq->tfds) goto error;
/* * iwl_pcie_tx_alloc - allocate TX context * Allocate all Tx DMA structures and initialize them
*/ staticint iwl_pcie_tx_alloc(struct iwl_trans *trans)
{ int ret; int txq_id, slots_num; struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
u16 bc_tbls_size = trans->mac_cfg->base->num_of_queues;
if (WARN_ON(trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)) return -EINVAL;
bc_tbls_size *= BC_TABLE_SIZE;
/*It is not allowed to alloc twice, so warn when this happens.
* We cannot rely on the previous allocation, so free and fail */ if (WARN_ON(trans_pcie->txq_memory)) {
ret = -EINVAL; goto error;
}
ret = iwl_pcie_alloc_dma_ptr(trans, &trans_pcie->txqs.scd_bc_tbls,
bc_tbls_size); if (ret) {
IWL_ERR(trans, "Scheduler BC Table allocation failed\n"); goto error;
}
int iwl_txq_init(struct iwl_trans *trans, struct iwl_txq *txq, int slots_num, bool cmd_queue)
{
u32 tfd_queue_max_size =
trans->mac_cfg->base->max_tfd_queue_size; int ret;
txq->need_update = false;
/* max_tfd_queue_size must be power-of-two size, otherwise * iwl_txq_inc_wrap and iwl_txq_dec_wrap are broken.
*/ if (WARN_ONCE(tfd_queue_max_size & (tfd_queue_max_size - 1), "Max tfd queue size must be a power of two, but is %d",
tfd_queue_max_size)) return -EINVAL;
/* Initialize queue's high/low-water marks, and head/tail indexes */
ret = iwl_queue_init(txq, slots_num); if (ret) return ret;
/* * Tell nic where to find circular buffer of TFDs for a * given Tx queue, and enable the DMA channel used for that * queue. * Circular buffer (TFD queue in DRAM) physical base address
*/
iwl_write_direct32(trans, FH_MEM_CBBC_QUEUE(trans, txq_id),
trans_pcie->txqs.txq[txq_id]->dma_addr >> 8);
}
iwl_set_bits_prph(trans, SCD_GP_CTRL, SCD_GP_CTRL_AUTO_ACTIVE_MODE); if (trans->mac_cfg->base->num_of_queues > 20)
iwl_set_bits_prph(trans, SCD_GP_CTRL,
SCD_GP_CTRL_ENABLE_31_QUEUES);
return 0;
error: /*Upon error, free only if we allocated something */ if (alloc)
iwl_pcie_tx_free(trans); return ret;
}
/* Make sure the NIC is still alive in the bus */ if (test_bit(STATUS_TRANS_DEAD, &trans->status)) return -ENODEV;
if (!trans->mac_cfg->base->apmg_wake_up_wa) return 0;
/* * wake up the NIC to make sure that the firmware will see the host * command - we will let the NIC sleep once all the host commands * returned. This needs to be done only on NICs that have * apmg_wake_up_wa set (see above.)
*/ if (!_iwl_trans_pcie_grab_nic_access(trans, false)) return -EIO;
/* * In iwl_trans_grab_nic_access(), we've acquired the reg_lock. * There, we also returned immediately if cmd_hold_nic_awake is * already true, so it's OK to unconditionally set it to true.
*/
trans_pcie->cmd_hold_nic_awake = true;
spin_unlock(&trans_pcie->reg_lock);
/* * station is asleep and we send data - that must * be uAPSD or PS-Poll. Don't rearm the timer.
*/ if (txq->frozen) return;
/* * if empty delete timer, otherwise move timer forward * since we're making progress on this queue
*/ if (txq->read_ptr == txq->write_ptr)
timer_delete(&txq->stuck_timer); else
mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout);
}
staticinlinebool iwl_txq_used(conststruct iwl_txq *q, int i, int read_ptr, int write_ptr)
{ int index = iwl_txq_get_cmd_index(q, i); int r = iwl_txq_get_cmd_index(q, read_ptr); int w = iwl_txq_get_cmd_index(q, write_ptr);
return w >= r ?
(index >= r && index < w) :
!(index < r && index >= w);
}
/* * iwl_pcie_cmdq_reclaim - Reclaim TX command queue entries already Tx'd * * When FW advances 'R' index, all entries between old and new 'R' index * need to be reclaimed. As result, some free space forms. If there is * enough free space (> low mark), wake the stack that feeds us.
*/ staticvoid iwl_pcie_cmdq_reclaim(struct iwl_trans *trans, int txq_id, int idx)
{ struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_txq *txq = trans_pcie->txqs.txq[txq_id]; int nfreed = 0;
u16 r;
lockdep_assert_held(&txq->lock);
idx = iwl_txq_get_cmd_index(txq, idx);
r = iwl_txq_get_cmd_index(txq, txq->read_ptr);
if (idx >= trans->mac_cfg->base->max_tfd_queue_size ||
(!iwl_txq_used(txq, idx, txq->read_ptr, txq->write_ptr))) {
WARN_ONCE(test_bit(txq_id, trans_pcie->txqs.queue_used), "%s: Read index for DMA queue txq id (%d), index %d is out of range [0-%d] %d %d.\n",
__func__, txq_id, idx,
trans->mac_cfg->base->max_tfd_queue_size,
txq->write_ptr, txq->read_ptr); return;
}
for (idx = iwl_txq_inc_wrap(trans, idx); r != idx;
r = iwl_txq_inc_wrap(trans, r)) {
txq->read_ptr = iwl_txq_inc_wrap(trans, txq->read_ptr);
if (nfreed++ > 0) {
IWL_ERR(trans, "HCMD skipped: index (%d) %d %d\n",
idx, txq->write_ptr, r);
iwl_force_nmi(trans);
}
}
if (txq->read_ptr == txq->write_ptr)
iwl_pcie_clear_cmd_in_flight(trans);
/* Receiver address (actually, Rx station's index into station table),
* combined with Traffic ID (QOS priority), in format used by Tx Scheduler */ #define BUILD_RAxTID(sta_id, tid) (((sta_id) << 4) + (tid))
if (test_and_set_bit(txq_id, trans_pcie->txqs.queue_used))
WARN_ONCE(1, "queue %d already used - expect issues", txq_id);
txq->wd_timeout = msecs_to_jiffies(wdg_timeout);
if (cfg) {
fifo = cfg->fifo;
/* Disable the scheduler prior configuring the cmd queue */ if (txq_id == trans->conf.cmd_queue &&
trans->conf.scd_set_active)
iwl_scd_enable_set_active(trans, 0);
/* Stop this Tx queue before configuring it */
iwl_scd_txq_set_inactive(trans, txq_id);
/* Set this queue as a chain-building queue unless it is CMD */ if (txq_id != trans->conf.cmd_queue)
iwl_scd_txq_set_chain(trans, txq_id);
if (cfg->aggregate) {
u16 ra_tid = BUILD_RAxTID(cfg->sta_id, cfg->tid);
/* Map receiver-address / traffic-ID to this queue */
iwl_pcie_txq_set_ratid_map(trans, ra_tid, txq_id);
/* enable aggregations for the queue */
iwl_scd_txq_enable_agg(trans, txq_id);
txq->ampdu = true;
} else { /* * disable aggregations for the queue, this will also * make the ra_tid mapping configuration irrelevant * since it is now a non-AGG queue.
*/
iwl_scd_txq_disable_agg(trans, txq_id);
ssn = txq->read_ptr;
}
} else { /* * If we need to move the SCD write pointer by steps of * 0x40, 0x80 or 0xc0, it gets stuck. Avoids this and let * the op_mode know by returning true later. * Do this only in case cfg is NULL since this trick can * be done only if we have DQA enabled which is true for mvm * only. And mvm never sets a cfg pointer. * This is really ugly, but this is the easiest way out for * this sad hardware issue. * This bug has been fixed on devices 9000 and up.
*/
scd_bug = !trans->mac_cfg->mq_rx_supported &&
!((ssn - txq->write_ptr) & 0x3f) &&
(ssn != txq->write_ptr); if (scd_bug)
ssn++;
}
/* Place first TFD at index corresponding to start sequence number.
* Assumes that ssn_idx is valid (!= 0xFFF) */
txq->read_ptr = (ssn & 0xff);
txq->write_ptr = (ssn & 0xff);
iwl_write_direct32(trans, HBUS_TARG_WRPTR,
(ssn & 0xff) | (txq_id << 8));
/* Set up Tx window size and frame limit for this queue */
iwl_trans_write_mem32(trans, trans_pcie->scd_base_addr +
SCD_CONTEXT_QUEUE_OFFSET(txq_id), 0);
iwl_trans_write_mem32(trans,
trans_pcie->scd_base_addr +
SCD_CONTEXT_QUEUE_OFFSET(txq_id) + sizeof(u32),
SCD_QUEUE_CTX_REG2_VAL(WIN_SIZE, frame_limit) |
SCD_QUEUE_CTX_REG2_VAL(FRAME_LIMIT, frame_limit));
/* Set up status area in SRAM, map to Tx DMA/FIFO, activate */
iwl_write_prph(trans, SCD_QUEUE_STATUS_BITS(txq_id),
(1 << SCD_QUEUE_STTS_REG_POS_ACTIVE) |
(cfg->fifo << SCD_QUEUE_STTS_REG_POS_TXF) |
(1 << SCD_QUEUE_STTS_REG_POS_WSL) |
SCD_QUEUE_STTS_REG_MSK);
/* enable the scheduler for this queue (only) */ if (txq_id == trans->conf.cmd_queue &&
trans->conf.scd_set_active)
iwl_scd_enable_set_active(trans, BIT(txq_id));
/* * Upon HW Rfkill - we stop the device, and then stop the queues * in the op_mode. Just for the sake of the simplicity of the op_mode, * allow the op_mode to call txq_disable after it already called * stop_device.
*/ if (!test_and_clear_bit(txq_id, trans_pcie->txqs.queue_used)) {
WARN_ONCE(test_bit(STATUS_DEVICE_ENABLED, &trans->status), "queue %d not used", txq_id); return;
}
if (configure_scd) {
iwl_scd_txq_set_inactive(trans, txq_id);
if (cmd->dataflags[i] & IWL_HCMD_DFL_NOCOPY) {
had_nocopy = true; if (WARN_ON(cmd->dataflags[i] & IWL_HCMD_DFL_DUP)) {
idx = -EINVAL; goto free_dup_buf;
}
} elseif (cmd->dataflags[i] & IWL_HCMD_DFL_DUP) { /* * This is also a chunk that isn't copied * to the static buffer so set had_nocopy.
*/
had_nocopy = true;
/* only allowed once */ if (WARN_ON(dup_buf)) {
idx = -EINVAL; goto free_dup_buf;
}
dup_buf = kmemdup(cmddata[i], cmdlen[i],
GFP_ATOMIC); if (!dup_buf) return -ENOMEM;
} else { /* NOCOPY must not be followed by normal! */ if (WARN_ON(had_nocopy)) {
idx = -EINVAL; goto free_dup_buf;
}
copy_size += cmdlen[i];
}
cmd_size += cmd->len[i];
}
/* * If any of the command structures end up being larger than * the TFD_MAX_PAYLOAD_SIZE and they aren't dynamically * allocated into separate TFDs, then we will need to * increase the size of the buffers.
*/ if (WARN(copy_size > TFD_MAX_PAYLOAD_SIZE, "Command %s (%#x) is too large (%d bytes)\n",
iwl_get_cmd_string(trans, cmd->id),
cmd->id, copy_size)) {
idx = -EINVAL; goto free_dup_buf;
}
/* re-initialize, this also marks the SG list as unused */
memset(out_meta, 0, sizeof(*out_meta)); if (cmd->flags & CMD_WANT_SKB)
out_meta->source = cmd;
/* set up the header */ if (group_id != 0) {
out_cmd->hdr_wide.cmd = iwl_cmd_opcode(cmd->id);
out_cmd->hdr_wide.group_id = group_id;
out_cmd->hdr_wide.version = iwl_cmd_version(cmd->id);
out_cmd->hdr_wide.length =
cpu_to_le16(cmd_size - sizeof(struct iwl_cmd_header_wide));
out_cmd->hdr_wide.reserved = 0;
out_cmd->hdr_wide.sequence =
cpu_to_le16(QUEUE_TO_SEQ(trans->conf.cmd_queue) |
INDEX_TO_SEQ(txq->write_ptr));
/* * Otherwise we need at least IWL_FIRST_TB_SIZE copied * in total (for bi-directional DMA), but copy up to what * we can fit into the payload for debug dump purposes.
*/
copy = min_t(int, TFD_MAX_PAYLOAD_SIZE - cmd_pos, cmd->len[i]);
/* * iwl_pcie_hcmd_complete - Pull unused buffers off the queue and reclaim them * @rxb: Rx buffer to reclaim
*/ void iwl_pcie_hcmd_complete(struct iwl_trans *trans, struct iwl_rx_cmd_buffer *rxb)
{ struct iwl_rx_packet *pkt = rxb_addr(rxb);
u16 sequence = le16_to_cpu(pkt->hdr.sequence);
u8 group_id;
u32 cmd_id; int txq_id = SEQ_TO_QUEUE(sequence); int index = SEQ_TO_INDEX(sequence); int cmd_index; struct iwl_device_cmd *cmd; struct iwl_cmd_meta *meta; struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_txq *txq = trans_pcie->txqs.txq[trans->conf.cmd_queue];
/* If a Tx command is being handled and it isn't in the actual * command queue then there a command routing bug has been introduced
* in the queue management code. */ if (IWL_FW_CHECK(trans, txq_id != trans->conf.cmd_queue, "wrong command queue %d (should be %d), sequence 0x%X readp=%d writep=%d pkt=%*phN\n",
txq_id, trans->conf.cmd_queue, sequence, txq->read_ptr,
txq->write_ptr, 32, pkt)) return;
/* set up the remaining entries to point to the data */ for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
dma_addr_t tb_phys; int tb_idx;
/* * Check if there's enough room on this page * * Note that we put a page chaining pointer *last* in the * page - we need it somewhere, and if it's there then we * avoid DMA mapping the last bits of the page which may * trigger the 32-bit boundary hardware bug. * * (see also get_workaround_page() in tx-gen2.c)
*/ if (((unsignedlong)p->pos & ~PAGE_MASK) + len < IWL_TSO_PAGE_DATA_SIZE) {
info = IWL_TSO_PAGE_INFO(page_address(p->page)); goto out;
}
/* We don't have enough room on this page, get a new one. */
iwl_pcie_free_and_unmap_tso_page(trans, p->page);
/* set the chaining pointer to NULL */
info->next = NULL;
/* Create a DMA mapping for the page */
phys = dma_map_page_attrs(trans->dev, p->page, 0, PAGE_SIZE,
DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); if (unlikely(dma_mapping_error(trans->dev, phys))) {
__free_page(p->page);
p->page = NULL;
return NULL;
}
/* Store physical address and set use count */
info->dma_addr = phys;
refcount_set(&info->use_count, 1);
out:
*page_ptr = p->page; /* Return an internal reference for the caller */
refcount_inc(&info->use_count);
ret = p->pos;
p->pos += len;
return ret;
}
/** * iwl_pcie_get_sgt_tb_phys - Find TB address in mapped SG list * @sgt: scatter gather table * @offset: Offset into the mapped memory (i.e. SKB payload data) * @len: Length of the area * * Find the DMA address that corresponds to the SKB payload data at the * position given by @offset. * * Returns: Address for TB entry
*/
dma_addr_t iwl_pcie_get_sgt_tb_phys(struct sg_table *sgt, unsignedint offset, unsignedint len)
{ struct scatterlist *sg; unsignedint sg_offset = 0; int i;
/* * Search the mapped DMA areas in the SG for the area that contains the * data at offset with the given length.
*/
for_each_sgtable_dma_sg(sgt, sg, i) { if (offset >= sg_offset &&
offset + len <= sg_offset + sg_dma_len(sg)) return sg_dma_address(sg) + offset - sg_offset;
sg_offset += sg_dma_len(sg);
}
WARN_ON_ONCE(1);
return DMA_MAPPING_ERROR;
}
/** * iwl_pcie_prep_tso - Prepare TSO page and SKB for sending * @trans: transport private data * @skb: the SKB to map * @cmd_meta: command meta to store the scatter list information for unmapping * @hdr: output argument for TSO headers * @hdr_room: requested length for TSO headers * @offset: offset into the data from which mapping should start * * Allocate space for a scatter gather list and TSO headers and map the SKB * using the scatter gather list. The SKB is unmapped again when the page is * free'ed again at the end of the operation. * * Returns: newly allocated and mapped scatter gather table with list
*/ struct sg_table *iwl_pcie_prep_tso(struct iwl_trans *trans, struct sk_buff *skb, struct iwl_cmd_meta *cmd_meta,
u8 **hdr, unsignedint hdr_room, unsignedint offset)
{ struct sg_table *sgt; unsignedint n_segments = skb_shinfo(skb)->nr_frags + 1; int orig_nents;
if (WARN_ON_ONCE(skb_has_frag_list(skb))) return NULL;
/* Only map the data, not the header (it is copied to the TSO page) */
orig_nents = skb_to_sgvec(skb, sgt->sgl, offset, skb->len - offset); if (WARN_ON_ONCE(orig_nents <= 0)) return NULL;
sgt->orig_nents = orig_nents;
/* And map the entire SKB */ if (dma_map_sgtable(trans->dev, sgt, DMA_TO_DEVICE, 0) < 0) return NULL;
/* Store non-zero (i.e. valid) offset for unmapping */
cmd_meta->sg_offset = (unsignedlong) sgt & ~PAGE_MASK;
/* if the packet is protected, then it must be CCMP or GCMP */
BUILD_BUG_ON(IEEE80211_CCMP_HDR_LEN != IEEE80211_GCMP_HDR_LEN);
iv_len = ieee80211_has_protected(hdr->frame_control) ?
IEEE80211_CCMP_HDR_LEN : 0;
/* total amount of header we may need for this A-MSDU */
hdr_room = DIV_ROUND_UP(total_len, mss) *
(3 + snap_ip_tcp_hdrlen + sizeof(struct ethhdr)) + iv_len;
/* Our device supports 9 segments at most, it will fit in 1 page */
sgt = iwl_pcie_prep_tso(trans, skb, out_meta, &start_hdr, hdr_room,
snap_ip_tcp_hdrlen + hdr_len + iv_len); if (!sgt) return -ENOMEM;
/* * Pull the ieee80211 header + IV to be able to use TSO core, * we will restore it for the tx_status flow.
*/
skb_pull(skb, hdr_len + iv_len);
/* * Remove the length of all the headers that we don't actually * have in the MPDU by themselves, but that we duplicate into * all the different MSDUs inside the A-MSDU.
*/
le16_add_cpu(&tx_cmd->params.len, -snap_ip_tcp_hdrlen);
tso_start(skb, &tso);
while (total_len) { /* this is the data left for this subframe */ unsignedint data_left =
min_t(unsignedint, mss, total_len); unsignedint hdr_tb_len;
dma_addr_t hdr_tb_phys;
u8 *subf_hdrs_start = pos_hdr;
iwl_pcie_txq_build_tfd(trans, txq, hdr_tb_phys,
hdr_tb_len, false);
trace_iwlwifi_dev_tx_tb(trans->dev, skb, start_hdr,
hdr_tb_phys, hdr_tb_len); /* add this subframe's headers' length to the tx_cmd */
le16_add_cpu(&tx_cmd->params.len, pos_hdr - subf_hdrs_start);
/* prepare the start_hdr for the next subframe */
start_hdr = pos_hdr;
/* put the payload */ while (data_left) { unsignedint size = min_t(unsignedint, tso.size,
data_left);
dma_addr_t tb_phys;
tb_phys = iwl_pcie_get_sgt_tb_phys(sgt, data_offset, size); /* Not a real mapping error, use direct comparison */ if (unlikely(tb_phys == DMA_MAPPING_ERROR)) return -EINVAL;
/* * iwl_txq_gen1_update_byte_cnt_tbl - Set up entry in Tx byte-count array
*/ staticvoid iwl_txq_gen1_update_byte_cnt_tbl(struct iwl_trans *trans, struct iwl_txq *txq, u16 byte_cnt, int num_tbs)
{ struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_bc_tbl_entry *scd_bc_tbl; int write_ptr = txq->write_ptr; int txq_id = txq->id;
u8 sec_ctl = 0;
u16 len = byte_cnt + IWL_TX_CRC_SIZE + IWL_TX_DELIMITER_SIZE;
__le16 bc_ent; struct iwl_device_tx_cmd *dev_cmd = txq->entries[txq->write_ptr].cmd; struct iwl_tx_cmd_v6 *tx_cmd = (void *)dev_cmd->payload;
u8 sta_id = tx_cmd->params.sta_id;
scd_bc_tbl = trans_pcie->txqs.scd_bc_tbls.addr;
sec_ctl = tx_cmd->params.sec_ctl;
switch (sec_ctl & TX_CMD_SEC_MSK) { case TX_CMD_SEC_CCM:
len += IEEE80211_CCMP_MIC_LEN; break; case TX_CMD_SEC_TKIP:
len += IEEE80211_TKIP_ICV_LEN; break; case TX_CMD_SEC_WEP:
len += IEEE80211_WEP_IV_LEN + IEEE80211_WEP_ICV_LEN; break;
}
if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_7000 &&
trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
len = DIV_ROUND_UP(len, 4);
if (WARN_ON(len > 0xFFF || write_ptr >= TFD_QUEUE_SIZE_MAX)) return;
if (WARN_ONCE(!test_bit(txq_id, trans_pcie->txqs.queue_used), "TX on unused queue %d\n", txq_id)) return -EINVAL;
if (skb_is_nonlinear(skb) &&
skb_shinfo(skb)->nr_frags > IWL_TRANS_PCIE_MAX_FRAGS(trans_pcie) &&
__skb_linearize(skb)) return -ENOMEM;
/* mac80211 always puts the full header into the SKB's head, * so there's no need to check if it's readable there
*/
hdr = (struct ieee80211_hdr *)skb->data;
fc = hdr->frame_control;
hdr_len = ieee80211_hdrlen(fc);
spin_lock(&txq->lock);
if (iwl_txq_space(trans, txq) < txq->high_mark) {
iwl_txq_stop(trans, txq);
/* don't put the packet on the ring, if there is no room */ if (unlikely(iwl_txq_space(trans, txq) < 3)) { struct iwl_device_tx_cmd **dev_cmd_ptr;
/* In AGG mode, the index in the ring must correspond to the WiFi * sequence number. This is a HW requirements to help the SCD to parse * the BA. * Check here that the packets are in the right place on the ring.
*/
wifi_seq = IEEE80211_SEQ_TO_SN(le16_to_cpu(hdr->seq_ctrl));
WARN_ONCE(txq->ampdu &&
(wifi_seq & 0xff) != txq->write_ptr, "Q: %d WiFi Seq %d tfdNum %d",
txq_id, wifi_seq, txq->write_ptr);
/* Set up driver data for this TFD */
txq->entries[txq->write_ptr].skb = skb;
txq->entries[txq->write_ptr].cmd = dev_cmd;
/* Set up first empty entry in queue's array of Tx/cmd buffers */
out_meta = &txq->entries[txq->write_ptr].meta;
memset(out_meta, 0, sizeof(*out_meta));
/* * The second TB (tb1) points to the remainder of the TX command * and the 802.11 header - dword aligned size * (This calculation modifies the TX command, so do it before the * setup of the first TB)
*/
len = sizeof(struct iwl_tx_cmd_v6) + sizeof(struct iwl_cmd_header) +
hdr_len - IWL_FIRST_TB_SIZE; /* do not align A-MSDU to dword as the subframe header aligns it */
amsdu = ieee80211_is_data_qos(fc) &&
(*ieee80211_get_qos_ctl(hdr) &
IEEE80211_QOS_CTL_A_MSDU_PRESENT); if (!amsdu) {
tb1_len = ALIGN(len, 4); /* Tell NIC about any 2-byte padding after MAC header */ if (tb1_len != len)
tx_cmd->params.tx_flags |= cpu_to_le32(TX_CMD_FLG_MH_PAD);
} else {
tb1_len = len;
}
/* * The first TB points to bi-directional DMA data, we'll * memcpy the data into it later.
*/
iwl_pcie_txq_build_tfd(trans, txq, tb0_phys,
IWL_FIRST_TB_SIZE, true);
/* there must be data left over for TB1 or this code must be changed */
BUILD_BUG_ON(sizeof(struct iwl_tx_cmd_v6) < IWL_FIRST_TB_SIZE);
BUILD_BUG_ON(sizeof(struct iwl_cmd_header) +
offsetofend(struct iwl_tx_cmd_v6_params, scratch) >
IWL_FIRST_TB_SIZE);
/* map the data for TB1 */
tb1_addr = ((u8 *)&dev_cmd->hdr) + IWL_FIRST_TB_SIZE;
tb1_phys = dma_map_single(trans->dev, tb1_addr, tb1_len, DMA_TO_DEVICE); if (unlikely(dma_mapping_error(trans->dev, tb1_phys))) goto out_err;
iwl_pcie_txq_build_tfd(trans, txq, tb1_phys, tb1_len, false);
/* * If gso_size wasn't set, don't give the frame "amsdu treatment" * (adding subframes, etc.). * This can happen in some testing flows when the amsdu was already * pre-built, and we just need to send the resulting skb.
*/ if (amsdu && skb_shinfo(skb)->gso_size) { if (unlikely(iwl_fill_data_tbs_amsdu(trans, skb, txq, hdr_len,
out_meta, dev_cmd,
tb1_len))) goto out_err;
} else { struct sk_buff *frag;
if (unlikely(iwl_fill_data_tbs(trans, skb, txq, hdr_len,
out_meta))) goto out_err;
/* building the A-MSDU might have changed this data, so memcpy it now */
memcpy(&txq->first_tb_bufs[txq->write_ptr], dev_cmd, IWL_FIRST_TB_SIZE);
tfd = iwl_txq_get_tfd(trans, txq, txq->write_ptr); /* Set up entry for this TFD in Tx byte-count array */
iwl_txq_gen1_update_byte_cnt_tbl(trans, txq, le16_to_cpu(tx_cmd->params.len),
iwl_txq_gen1_tfd_get_num_tbs(tfd));
wait_write_ptr = ieee80211_has_morefrags(fc);
/* start timer if queue currently empty */ if (txq->read_ptr == txq->write_ptr && txq->wd_timeout) { /* * If the TXQ is active, then set the timer, if not, * set the timer in remainder so that the timer will * be armed with the right value when the station will * wake up.
*/ if (!txq->frozen)
mod_timer(&txq->stuck_timer,
jiffies + txq->wd_timeout); else
txq->frozen_expiry_remainder = txq->wd_timeout;
}
/* Tell device the write index *just past* this latest filled TFD */
txq->write_ptr = iwl_txq_inc_wrap(trans, txq->write_ptr); if (!wait_write_ptr)
iwl_pcie_txq_inc_wr_ptr(trans, txq);
/* * At this point the frame is "transmitted" successfully * and we will get a TX status notification eventually.
*/
spin_unlock(&txq->lock); return 0;
out_err:
iwl_txq_gen1_tfd_unmap(trans, out_meta, txq, txq->write_ptr);
spin_unlock(&txq->lock); return -1;
}
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.
