/******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * * Copyright (C) 2017-2025 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * * www.broadcom.com * * Portions Copyright (C) 2004-2005 Christoph Hellwig * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of version 2 of the GNU General * * Public License as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful. * * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND * * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, * * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE * * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD * * TO BE LEGALLY INVALID. See the GNU General Public License for * * more details, a copy of which can be found in the file COPYING * * included with this package. *
*******************************************************************/
/* There are only four IOCB completion types. */ typedefenum _lpfc_iocb_type {
LPFC_UNKNOWN_IOCB,
LPFC_UNSOL_IOCB,
LPFC_SOL_IOCB,
LPFC_ABORT_IOCB
} lpfc_iocb_type;
#ifdefined(CONFIG_64BIT) && defined(__LITTLE_ENDIAN) /** * lpfc_sli4_pcimem_bcopy - SLI4 memory copy function * @srcp: Source memory pointer. * @destp: Destination memory pointer. * @cnt: Number of words required to be copied. * Must be a multiple of sizeof(uint64_t) * * This function is used for copying data between driver memory * and the SLI WQ. This function also changes the endianness * of each word if native endianness is different from SLI * endianness. This function can be called with or without * lock.
**/ staticvoid
lpfc_sli4_pcimem_bcopy(void *srcp, void *destp, uint32_t cnt)
{
uint64_t *src = srcp;
uint64_t *dest = destp; int i;
for (i = 0; i < (int)cnt; i += sizeof(uint64_t))
*dest++ = *src++;
} #else #define lpfc_sli4_pcimem_bcopy(a, b, c) lpfc_sli_pcimem_bcopy(a, b, c) #endif
/** * lpfc_sli4_wq_put - Put a Work Queue Entry on an Work Queue * @q: The Work Queue to operate on. * @wqe: The work Queue Entry to put on the Work queue. * * This routine will copy the contents of @wqe to the next available entry on * the @q. This function will then ring the Work Queue Doorbell to signal the * HBA to start processing the Work Queue Entry. This function returns 0 if * successful. If no entries are available on @q then this function will return * -ENOMEM. * The caller is expected to hold the hbalock when calling this routine.
**/ staticint
lpfc_sli4_wq_put(struct lpfc_queue *q, union lpfc_wqe128 *wqe)
{ union lpfc_wqe *temp_wqe; struct lpfc_register doorbell;
uint32_t host_index;
uint32_t idx;
uint32_t i = 0;
uint8_t *tmp;
u32 if_type;
/* sanity check on queue memory */ if (unlikely(!q)) return -ENOMEM;
temp_wqe = lpfc_sli4_qe(q, q->host_index);
/* If the host has not yet processed the next entry then we are done */
idx = ((q->host_index + 1) % q->entry_count); if (idx == q->hba_index) {
q->WQ_overflow++; return -EBUSY;
}
q->WQ_posted++; /* set consumption flag every once in a while */ if (!((q->host_index + 1) % q->notify_interval))
bf_set(wqe_wqec, &wqe->generic.wqe_com, 1); else
bf_set(wqe_wqec, &wqe->generic.wqe_com, 0); if (q->phba->sli3_options & LPFC_SLI4_PHWQ_ENABLED)
bf_set(wqe_wqid, &wqe->generic.wqe_com, q->queue_id);
lpfc_sli4_pcimem_bcopy(wqe, temp_wqe, q->entry_size); if (q->dpp_enable && q->phba->cfg_enable_dpp) { /* write to DPP aperture taking advatage of Combined Writes */
tmp = (uint8_t *)temp_wqe; #ifdef __raw_writeq for (i = 0; i < q->entry_size; i += sizeof(uint64_t))
__raw_writeq(*((uint64_t *)(tmp + i)),
q->dpp_regaddr + i); #else for (i = 0; i < q->entry_size; i += sizeof(uint32_t))
__raw_writel(*((uint32_t *)(tmp + i)),
q->dpp_regaddr + i); #endif
} /* ensure WQE bcopy and DPP flushed before doorbell write */
wmb();
/* Update the host index before invoking device */
host_index = q->host_index;
/** * lpfc_sli4_wq_release - Updates internal hba index for WQ * @q: The Work Queue to operate on. * @index: The index to advance the hba index to. * * This routine will update the HBA index of a queue to reflect consumption of * Work Queue Entries by the HBA. When the HBA indicates that it has consumed * an entry the host calls this function to update the queue's internal * pointers.
**/ staticvoid
lpfc_sli4_wq_release(struct lpfc_queue *q, uint32_t index)
{ /* sanity check on queue memory */ if (unlikely(!q)) return;
q->hba_index = index;
}
/** * lpfc_sli4_mq_put - Put a Mailbox Queue Entry on an Mailbox Queue * @q: The Mailbox Queue to operate on. * @mqe: The Mailbox Queue Entry to put on the Work queue. * * This routine will copy the contents of @mqe to the next available entry on * the @q. This function will then ring the Work Queue Doorbell to signal the * HBA to start processing the Work Queue Entry. This function returns 0 if * successful. If no entries are available on @q then this function will return * -ENOMEM. * The caller is expected to hold the hbalock when calling this routine.
**/ static uint32_t
lpfc_sli4_mq_put(struct lpfc_queue *q, struct lpfc_mqe *mqe)
{ struct lpfc_mqe *temp_mqe; struct lpfc_register doorbell;
/* sanity check on queue memory */ if (unlikely(!q)) return -ENOMEM;
temp_mqe = lpfc_sli4_qe(q, q->host_index);
/* If the host has not yet processed the next entry then we are done */ if (((q->host_index + 1) % q->entry_count) == q->hba_index) return -ENOMEM;
lpfc_sli4_pcimem_bcopy(mqe, temp_mqe, q->entry_size); /* Save off the mailbox pointer for completion */
q->phba->mbox = (MAILBOX_t *)temp_mqe;
/* Update the host index before invoking device */
q->host_index = ((q->host_index + 1) % q->entry_count);
/** * lpfc_sli4_mq_release - Updates internal hba index for MQ * @q: The Mailbox Queue to operate on. * * This routine will update the HBA index of a queue to reflect consumption of * a Mailbox Queue Entry by the HBA. When the HBA indicates that it has consumed * an entry the host calls this function to update the queue's internal * pointers. This routine returns the number of entries that were consumed by * the HBA.
**/ static uint32_t
lpfc_sli4_mq_release(struct lpfc_queue *q)
{ /* sanity check on queue memory */ if (unlikely(!q)) return 0;
/** * lpfc_sli4_eq_get - Gets the next valid EQE from a EQ * @q: The Event Queue to get the first valid EQE from * * This routine will get the first valid Event Queue Entry from @q, update * the queue's internal hba index, and return the EQE. If no valid EQEs are in * the Queue (no more work to do), or the Queue is full of EQEs that have been * processed, but not popped back to the HBA then this routine will return NULL.
**/ staticstruct lpfc_eqe *
lpfc_sli4_eq_get(struct lpfc_queue *q)
{ struct lpfc_eqe *eqe;
/* sanity check on queue memory */ if (unlikely(!q)) return NULL;
eqe = lpfc_sli4_qe(q, q->host_index);
/* If the next EQE is not valid then we are done */ if (bf_get_le32(lpfc_eqe_valid, eqe) != q->qe_valid) return NULL;
/* * insert barrier for instruction interlock : data from the hardware * must have the valid bit checked before it can be copied and acted * upon. Speculative instructions were allowing a bcopy at the start * of lpfc_sli4_fp_handle_wcqe(), which is called immediately * after our return, to copy data before the valid bit check above * was done. As such, some of the copied data was stale. The barrier * ensures the check is before any data is copied.
*/
mb(); return eqe;
}
/** * lpfc_sli4_eq_clr_intr - Turn off interrupts from this EQ * @q: The Event Queue to disable interrupts *
**/ void
lpfc_sli4_eq_clr_intr(struct lpfc_queue *q)
{ struct lpfc_register doorbell;
/** * lpfc_sli4_write_eq_db - write EQ DB for eqe's consumed or arm state * @phba: adapter with EQ * @q: The Event Queue that the host has completed processing for. * @count: Number of elements that have been consumed * @arm: Indicates whether the host wants to arms this CQ. * * This routine will notify the HBA, by ringing the doorbell, that count * number of EQEs have been processed. The @arm parameter indicates whether * the queue should be rearmed when ringing the doorbell.
**/ void
lpfc_sli4_write_eq_db(struct lpfc_hba *phba, struct lpfc_queue *q,
uint32_t count, bool arm)
{ struct lpfc_register doorbell;
/* sanity check on queue memory */ if (unlikely(!q || (count == 0 && !arm))) return;
/* ring doorbell for number popped */
doorbell.word0 = 0; if (arm) {
bf_set(lpfc_eqcq_doorbell_arm, &doorbell, 1);
bf_set(lpfc_eqcq_doorbell_eqci, &doorbell, 1);
}
bf_set(lpfc_eqcq_doorbell_num_released, &doorbell, count);
bf_set(lpfc_eqcq_doorbell_qt, &doorbell, LPFC_QUEUE_TYPE_EVENT);
bf_set(lpfc_eqcq_doorbell_eqid_hi, &doorbell,
(q->queue_id >> LPFC_EQID_HI_FIELD_SHIFT));
bf_set(lpfc_eqcq_doorbell_eqid_lo, &doorbell, q->queue_id);
writel(doorbell.word0, q->phba->sli4_hba.EQDBregaddr); /* PCI read to flush PCI pipeline on re-arming for INTx mode */ if ((q->phba->intr_type == INTx) && (arm == LPFC_QUEUE_REARM))
readl(q->phba->sli4_hba.EQDBregaddr);
}
/** * lpfc_sli4_if6_write_eq_db - write EQ DB for eqe's consumed or arm state * @phba: adapter with EQ * @q: The Event Queue that the host has completed processing for. * @count: Number of elements that have been consumed * @arm: Indicates whether the host wants to arms this CQ. * * This routine will notify the HBA, by ringing the doorbell, that count * number of EQEs have been processed. The @arm parameter indicates whether * the queue should be rearmed when ringing the doorbell.
**/ void
lpfc_sli4_if6_write_eq_db(struct lpfc_hba *phba, struct lpfc_queue *q,
uint32_t count, bool arm)
{ struct lpfc_register doorbell;
/* sanity check on queue memory */ if (unlikely(!q || (count == 0 && !arm))) return;
/* ring doorbell for number popped */
doorbell.word0 = 0; if (arm)
bf_set(lpfc_if6_eq_doorbell_arm, &doorbell, 1);
bf_set(lpfc_if6_eq_doorbell_num_released, &doorbell, count);
bf_set(lpfc_if6_eq_doorbell_eqid, &doorbell, q->queue_id);
writel(doorbell.word0, q->phba->sli4_hba.EQDBregaddr); /* PCI read to flush PCI pipeline on re-arming for INTx mode */ if ((q->phba->intr_type == INTx) && (arm == LPFC_QUEUE_REARM))
readl(q->phba->sli4_hba.EQDBregaddr);
}
/* if the index wrapped around, toggle the valid bit */ if (phba->sli4_hba.pc_sli4_params.eqav && !eq->host_index)
eq->qe_valid = (eq->qe_valid) ? 0 : 1;
}
/* walk all the EQ entries and drop on the floor */
eqe = lpfc_sli4_eq_get(eq); while (eqe) { /* Get the reference to the corresponding CQ */
cqid = bf_get_le32(lpfc_eqe_resource_id, eqe);
cq = NULL;
list_for_each_entry(childq, &eq->child_list, list) { if (childq->queue_id == cqid) {
cq = childq; break;
}
} /* If CQ is valid, iterate through it and drop all the CQEs */ if (cq) {
cqe = lpfc_sli4_cq_get(cq); while (cqe) {
__lpfc_sli4_consume_cqe(phba, cq, cqe);
cq_count++;
cqe = lpfc_sli4_cq_get(cq);
} /* Clear and re-arm the CQ */
phba->sli4_hba.sli4_write_cq_db(phba, cq, cq_count,
LPFC_QUEUE_REARM);
cq_count = 0;
}
__lpfc_sli4_consume_eqe(phba, eq, eqe);
eq_count++;
eqe = lpfc_sli4_eq_get(eq);
}
/* Clear and re-arm the EQ */
phba->sli4_hba.sli4_write_eq_db(phba, eq, eq_count, LPFC_QUEUE_REARM);
}
/** * lpfc_sli4_cq_get - Gets the next valid CQE from a CQ * @q: The Completion Queue to get the first valid CQE from * * This routine will get the first valid Completion Queue Entry from @q, update * the queue's internal hba index, and return the CQE. If no valid CQEs are in * the Queue (no more work to do), or the Queue is full of CQEs that have been * processed, but not popped back to the HBA then this routine will return NULL.
**/ staticstruct lpfc_cqe *
lpfc_sli4_cq_get(struct lpfc_queue *q)
{ struct lpfc_cqe *cqe;
/* sanity check on queue memory */ if (unlikely(!q)) return NULL;
cqe = lpfc_sli4_qe(q, q->host_index);
/* If the next CQE is not valid then we are done */ if (bf_get_le32(lpfc_cqe_valid, cqe) != q->qe_valid) return NULL;
/* * insert barrier for instruction interlock : data from the hardware * must have the valid bit checked before it can be copied and acted * upon. Given what was seen in lpfc_sli4_cq_get() of speculative * instructions allowing action on content before valid bit checked, * add barrier here as well. May not be needed as "content" is a * single 32-bit entity here (vs multi word structure for cq's).
*/
mb(); return cqe;
}
/* if the index wrapped around, toggle the valid bit */ if (phba->sli4_hba.pc_sli4_params.cqav && !cq->host_index)
cq->qe_valid = (cq->qe_valid) ? 0 : 1;
}
/** * lpfc_sli4_write_cq_db - write cq DB for entries consumed or arm state. * @phba: the adapter with the CQ * @q: The Completion Queue that the host has completed processing for. * @count: the number of elements that were consumed * @arm: Indicates whether the host wants to arms this CQ. * * This routine will notify the HBA, by ringing the doorbell, that the * CQEs have been processed. The @arm parameter specifies whether the * queue should be rearmed when ringing the doorbell.
**/ void
lpfc_sli4_write_cq_db(struct lpfc_hba *phba, struct lpfc_queue *q,
uint32_t count, bool arm)
{ struct lpfc_register doorbell;
/* sanity check on queue memory */ if (unlikely(!q || (count == 0 && !arm))) return;
/* ring doorbell for number popped */
doorbell.word0 = 0; if (arm)
bf_set(lpfc_eqcq_doorbell_arm, &doorbell, 1);
bf_set(lpfc_eqcq_doorbell_num_released, &doorbell, count);
bf_set(lpfc_eqcq_doorbell_qt, &doorbell, LPFC_QUEUE_TYPE_COMPLETION);
bf_set(lpfc_eqcq_doorbell_cqid_hi, &doorbell,
(q->queue_id >> LPFC_CQID_HI_FIELD_SHIFT));
bf_set(lpfc_eqcq_doorbell_cqid_lo, &doorbell, q->queue_id);
writel(doorbell.word0, q->phba->sli4_hba.CQDBregaddr);
}
/** * lpfc_sli4_if6_write_cq_db - write cq DB for entries consumed or arm state. * @phba: the adapter with the CQ * @q: The Completion Queue that the host has completed processing for. * @count: the number of elements that were consumed * @arm: Indicates whether the host wants to arms this CQ. * * This routine will notify the HBA, by ringing the doorbell, that the * CQEs have been processed. The @arm parameter specifies whether the * queue should be rearmed when ringing the doorbell.
**/ void
lpfc_sli4_if6_write_cq_db(struct lpfc_hba *phba, struct lpfc_queue *q,
uint32_t count, bool arm)
{ struct lpfc_register doorbell;
/* sanity check on queue memory */ if (unlikely(!q || (count == 0 && !arm))) return;
/* ring doorbell for number popped */
doorbell.word0 = 0; if (arm)
bf_set(lpfc_if6_cq_doorbell_arm, &doorbell, 1);
bf_set(lpfc_if6_cq_doorbell_num_released, &doorbell, count);
bf_set(lpfc_if6_cq_doorbell_cqid, &doorbell, q->queue_id);
writel(doorbell.word0, q->phba->sli4_hba.CQDBregaddr);
}
/* * lpfc_sli4_rq_put - Put a Receive Buffer Queue Entry on a Receive Queue * * This routine will copy the contents of @wqe to the next available entry on * the @q. This function will then ring the Receive Queue Doorbell to signal the * HBA to start processing the Receive Queue Entry. This function returns the * index that the rqe was copied to if successful. If no entries are available * on @q then this function will return -ENOMEM. * The caller is expected to hold the hbalock when calling this routine.
**/ int
lpfc_sli4_rq_put(struct lpfc_queue *hq, struct lpfc_queue *dq, struct lpfc_rqe *hrqe, struct lpfc_rqe *drqe)
{ struct lpfc_rqe *temp_hrqe; struct lpfc_rqe *temp_drqe; struct lpfc_register doorbell; int hq_put_index; int dq_put_index;
if (hq->type != LPFC_HRQ || dq->type != LPFC_DRQ) return -EINVAL; if (hq_put_index != dq_put_index) return -EINVAL; /* If the host has not yet processed the next entry then we are done */ if (((hq_put_index + 1) % hq->entry_count) == hq->hba_index) return -EBUSY;
lpfc_sli4_pcimem_bcopy(hrqe, temp_hrqe, hq->entry_size);
lpfc_sli4_pcimem_bcopy(drqe, temp_drqe, dq->entry_size);
/* Update the host index to point to the next slot */
hq->host_index = ((hq_put_index + 1) % hq->entry_count);
dq->host_index = ((dq_put_index + 1) % dq->entry_count);
hq->RQ_buf_posted++;
/* * lpfc_sli4_rq_release - Updates internal hba index for RQ * * This routine will update the HBA index of a queue to reflect consumption of * one Receive Queue Entry by the HBA. When the HBA indicates that it has * consumed an entry the host calls this function to update the queue's * internal pointers. This routine returns the number of entries that were * consumed by the HBA.
**/ static uint32_t
lpfc_sli4_rq_release(struct lpfc_queue *hq, struct lpfc_queue *dq)
{ /* sanity check on queue memory */ if (unlikely(!hq) || unlikely(!dq)) return 0;
/** * lpfc_cmd_iocb - Get next command iocb entry in the ring * @phba: Pointer to HBA context object. * @pring: Pointer to driver SLI ring object. * * This function returns pointer to next command iocb entry * in the command ring. The caller must hold hbalock to prevent * other threads consume the next command iocb. * SLI-2/SLI-3 provide different sized iocbs.
**/ staticinline IOCB_t *
lpfc_cmd_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
{ return (IOCB_t *) (((char *) pring->sli.sli3.cmdringaddr) +
pring->sli.sli3.cmdidx * phba->iocb_cmd_size);
}
/** * lpfc_resp_iocb - Get next response iocb entry in the ring * @phba: Pointer to HBA context object. * @pring: Pointer to driver SLI ring object. * * This function returns pointer to next response iocb entry * in the response ring. The caller must hold hbalock to make sure * that no other thread consume the next response iocb. * SLI-2/SLI-3 provide different sized iocbs.
**/ staticinline IOCB_t *
lpfc_resp_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
{ return (IOCB_t *) (((char *) pring->sli.sli3.rspringaddr) +
pring->sli.sli3.rspidx * phba->iocb_rsp_size);
}
/** * __lpfc_sli_get_iocbq - Allocates an iocb object from iocb pool * @phba: Pointer to HBA context object. * * This function is called with hbalock held. This function * allocates a new driver iocb object from the iocb pool. If the * allocation is successful, it returns pointer to the newly * allocated iocb object else it returns NULL.
**/ struct lpfc_iocbq *
__lpfc_sli_get_iocbq(struct lpfc_hba *phba)
{ struct list_head *lpfc_iocb_list = &phba->lpfc_iocb_list; struct lpfc_iocbq * iocbq = NULL;
lockdep_assert_held(&phba->hbalock);
list_remove_head(lpfc_iocb_list, iocbq, struct lpfc_iocbq, list); if (iocbq)
phba->iocb_cnt++; if (phba->iocb_cnt > phba->iocb_max)
phba->iocb_max = phba->iocb_cnt; return iocbq;
}
/** * __lpfc_clear_active_sglq - Remove the active sglq for this XRI. * @phba: Pointer to HBA context object. * @xritag: XRI value. * * This function clears the sglq pointer from the array of active * sglq's. The xritag that is passed in is used to index into the * array. Before the xritag can be used it needs to be adjusted * by subtracting the xribase. * * Returns sglq ponter = success, NULL = Failure.
**/ struct lpfc_sglq *
__lpfc_clear_active_sglq(struct lpfc_hba *phba, uint16_t xritag)
{ struct lpfc_sglq *sglq;
/** * __lpfc_get_active_sglq - Get the active sglq for this XRI. * @phba: Pointer to HBA context object. * @xritag: XRI value. * * This function returns the sglq pointer from the array of active * sglq's. The xritag that is passed in is used to index into the * array. Before the xritag can be used it needs to be adjusted * by subtracting the xribase. * * Returns sglq ponter = success, NULL = Failure.
**/ struct lpfc_sglq *
__lpfc_get_active_sglq(struct lpfc_hba *phba, uint16_t xritag)
{ struct lpfc_sglq *sglq;
/** * lpfc_clr_rrq_active - Clears RRQ active bit in xri_bitmap. * @phba: Pointer to HBA context object. * @xritag: xri used in this exchange. * @rrq: The RRQ to be cleared. *
**/ void
lpfc_clr_rrq_active(struct lpfc_hba *phba,
uint16_t xritag, struct lpfc_node_rrq *rrq)
{ struct lpfc_nodelist *ndlp = NULL;
/* Lookup did to verify if did is still active on this vport */ if (rrq->vport)
ndlp = lpfc_findnode_did(rrq->vport, rrq->nlp_DID);
/** * lpfc_handle_rrq_active - Checks if RRQ has waithed RATOV. * @phba: Pointer to HBA context object. * * This function is called with hbalock held. This function * Checks if stop_time (ratov from setting rrq active) has * been reached, if it has and the send_rrq flag is set then * it will call lpfc_send_rrq. If the send_rrq flag is not set * then it will just call the routine to clear the rrq and * free the rrq resource. * The timer is set to the next rrq that is going to expire before * leaving the routine. *
**/ void
lpfc_handle_rrq_active(struct lpfc_hba *phba)
{ struct lpfc_node_rrq *rrq; struct lpfc_node_rrq *nextrrq; unsignedlong next_time; unsignedlong iflags;
LIST_HEAD(send_rrq);
clear_bit(HBA_RRQ_ACTIVE, &phba->hba_flag);
next_time = jiffies + secs_to_jiffies(phba->fc_ratov + 1);
spin_lock_irqsave(&phba->rrq_list_lock, iflags);
list_for_each_entry_safe(rrq, nextrrq,
&phba->active_rrq_list, list) { if (time_after(jiffies, rrq->rrq_stop_time))
list_move(&rrq->list, &send_rrq); elseif (time_before(rrq->rrq_stop_time, next_time))
next_time = rrq->rrq_stop_time;
}
spin_unlock_irqrestore(&phba->rrq_list_lock, iflags); if ((!list_empty(&phba->active_rrq_list)) &&
(!test_bit(FC_UNLOADING, &phba->pport->load_flag)))
mod_timer(&phba->rrq_tmr, next_time);
list_for_each_entry_safe(rrq, nextrrq, &send_rrq, list) {
list_del(&rrq->list); if (!rrq->send_rrq) { /* this call will free the rrq */
lpfc_clr_rrq_active(phba, rrq->xritag, rrq);
} elseif (lpfc_send_rrq(phba, rrq)) { /* if we send the rrq then the completion handler * will clear the bit in the xribitmap.
*/
lpfc_clr_rrq_active(phba, rrq->xritag,
rrq);
}
}
}
/** * lpfc_get_active_rrq - Get the active RRQ for this exchange. * @vport: Pointer to vport context object. * @xri: The xri used in the exchange. * @did: The targets DID for this exchange. * * returns NULL = rrq not found in the phba->active_rrq_list. * rrq = rrq for this xri and target.
**/ struct lpfc_node_rrq *
lpfc_get_active_rrq(struct lpfc_vport *vport, uint16_t xri, uint32_t did)
{ struct lpfc_hba *phba = vport->phba; struct lpfc_node_rrq *rrq; struct lpfc_node_rrq *nextrrq; unsignedlong iflags;
/** * lpfc_cleanup_vports_rrqs - Remove and clear the active RRQ for this vport. * @vport: Pointer to vport context object. * @ndlp: Pointer to the lpfc_node_list structure. * If ndlp is NULL Remove all active RRQs for this vport from the * phba->active_rrq_list and clear the rrq. * If ndlp is not NULL then only remove rrqs for this vport & this ndlp.
**/ void
lpfc_cleanup_vports_rrqs(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
/** * lpfc_test_rrq_active - Test RRQ bit in xri_bitmap. * @phba: Pointer to HBA context object. * @ndlp: Targets nodelist pointer for this exchange. * @xritag: the xri in the bitmap to test. * * This function returns: * 0 = rrq not active for this xri * 1 = rrq is valid for this xri.
**/ int
lpfc_test_rrq_active(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
uint16_t xritag)
{ if (!ndlp) return 0; if (!ndlp->active_rrqs_xri_bitmap) return 0; if (test_bit(xritag, ndlp->active_rrqs_xri_bitmap)) return 1; else return 0;
}
/** * lpfc_set_rrq_active - set RRQ active bit in xri_bitmap. * @phba: Pointer to HBA context object. * @ndlp: nodelist pointer for this target. * @xritag: xri used in this exchange. * @rxid: Remote Exchange ID. * @send_rrq: Flag used to determine if we should send rrq els cmd. * * This function takes the hbalock. * The active bit is always set in the active rrq xri_bitmap even * if there is no slot avaiable for the other rrq information. * * returns 0 rrq actived for this xri * < 0 No memory or invalid ndlp.
**/ int
lpfc_set_rrq_active(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
uint16_t xritag, uint16_t rxid, uint16_t send_rrq)
{ unsignedlong iflags; struct lpfc_node_rrq *rrq; int empty;
if (!ndlp) return -EINVAL;
if (!phba->cfg_enable_rrq) return -EINVAL;
if (test_bit(FC_UNLOADING, &phba->pport->load_flag)) {
clear_bit(HBA_RRQ_ACTIVE, &phba->hba_flag); goto outnl;
}
spin_lock_irqsave(&phba->hbalock, iflags); if (ndlp->vport && test_bit(FC_UNLOADING, &ndlp->vport->load_flag)) goto out;
if (!ndlp->active_rrqs_xri_bitmap) goto out;
if (test_and_set_bit(xritag, ndlp->active_rrqs_xri_bitmap)) goto out;
/** * __lpfc_sli_get_els_sglq - Allocates an iocb object from sgl pool * @phba: Pointer to HBA context object. * @piocbq: Pointer to the iocbq. * * The driver calls this function with either the nvme ls ring lock * or the fc els ring lock held depending on the iocb usage. This function * gets a new driver sglq object from the sglq list. If the list is not empty * then it is successful, it returns pointer to the newly allocated sglq * object else it returns NULL.
**/ staticstruct lpfc_sglq *
__lpfc_sli_get_els_sglq(struct lpfc_hba *phba, struct lpfc_iocbq *piocbq)
{ struct list_head *lpfc_els_sgl_list = &phba->sli4_hba.lpfc_els_sgl_list; struct lpfc_sglq *sglq = NULL; struct lpfc_sglq *start_sglq = NULL; struct lpfc_io_buf *lpfc_cmd; struct lpfc_nodelist *ndlp; int found = 0;
u8 cmnd;
spin_lock(&phba->sli4_hba.sgl_list_lock);
list_remove_head(lpfc_els_sgl_list, sglq, struct lpfc_sglq, list);
start_sglq = sglq; while (!found) { if (!sglq) break; if (ndlp && ndlp->active_rrqs_xri_bitmap &&
test_bit(sglq->sli4_lxritag,
ndlp->active_rrqs_xri_bitmap)) { /* This xri has an rrq outstanding for this DID. * put it back in the list and get another xri.
*/
list_add_tail(&sglq->list, lpfc_els_sgl_list);
sglq = NULL;
list_remove_head(lpfc_els_sgl_list, sglq, struct lpfc_sglq, list); if (sglq == start_sglq) {
list_add_tail(&sglq->list, lpfc_els_sgl_list);
sglq = NULL; break;
} else continue;
}
sglq->ndlp = ndlp;
found = 1;
phba->sli4_hba.lpfc_sglq_active_list[sglq->sli4_lxritag] = sglq;
sglq->state = SGL_ALLOCATED;
}
spin_unlock(&phba->sli4_hba.sgl_list_lock); return sglq;
}
/** * __lpfc_sli_get_nvmet_sglq - Allocates an iocb object from sgl pool * @phba: Pointer to HBA context object. * @piocbq: Pointer to the iocbq. * * This function is called with the sgl_list lock held. This function * gets a new driver sglq object from the sglq list. If the * list is not empty then it is successful, it returns pointer to the newly * allocated sglq object else it returns NULL.
**/ struct lpfc_sglq *
__lpfc_sli_get_nvmet_sglq(struct lpfc_hba *phba, struct lpfc_iocbq *piocbq)
{ struct list_head *lpfc_nvmet_sgl_list; struct lpfc_sglq *sglq = NULL;
/** * lpfc_sli_get_iocbq - Allocates an iocb object from iocb pool * @phba: Pointer to HBA context object. * * This function is called with no lock held. This function * allocates a new driver iocb object from the iocb pool. If the * allocation is successful, it returns pointer to the newly * allocated iocb object else it returns NULL.
**/ struct lpfc_iocbq *
lpfc_sli_get_iocbq(struct lpfc_hba *phba)
{ struct lpfc_iocbq * iocbq = NULL; unsignedlong iflags;
/** * __lpfc_sli_release_iocbq_s4 - Release iocb to the iocb pool * @phba: Pointer to HBA context object. * @iocbq: Pointer to driver iocb object. * * This function is called to release the driver iocb object * to the iocb pool. The iotag in the iocb object * does not change for each use of the iocb object. This function * clears all other fields of the iocb object when it is freed. * The sqlq structure that holds the xritag and phys and virtual * mappings for the scatter gather list is retrieved from the * active array of sglq. The get of the sglq pointer also clears * the entry in the array. If the status of the IO indiactes that * this IO was aborted then the sglq entry it put on the * lpfc_abts_els_sgl_list until the CQ_ABORTED_XRI is received. If the * IO has good status or fails for any other reason then the sglq * entry is added to the free list (lpfc_els_sgl_list). The hbalock is * asserted held in the code path calling this routine.
**/ staticvoid
__lpfc_sli_release_iocbq_s4(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
{ struct lpfc_sglq *sglq; unsignedlong iflag = 0; struct lpfc_sli_ring *pring;
/** * __lpfc_sli_release_iocbq_s3 - Release iocb to the iocb pool * @phba: Pointer to HBA context object. * @iocbq: Pointer to driver iocb object. * * This function is called to release the driver iocb object to the * iocb pool. The iotag in the iocb object does not change for each * use of the iocb object. This function clears all other fields of * the iocb object when it is freed. The hbalock is asserted held in * the code path calling this routine.
**/ staticvoid
__lpfc_sli_release_iocbq_s3(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
{
/* * Clean all volatile data fields, preserve iotag and node struct.
*/
memset_startat(iocbq, 0, iocb);
iocbq->sli4_xritag = NO_XRI;
list_add_tail(&iocbq->list, &phba->lpfc_iocb_list);
}
/** * __lpfc_sli_release_iocbq - Release iocb to the iocb pool * @phba: Pointer to HBA context object. * @iocbq: Pointer to driver iocb object. * * This function is called with hbalock held to release driver * iocb object to the iocb pool. The iotag in the iocb object * does not change for each use of the iocb object. This function * clears all other fields of the iocb object when it is freed.
**/ staticvoid
__lpfc_sli_release_iocbq(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
{
lockdep_assert_held(&phba->hbalock);
/** * lpfc_sli_release_iocbq - Release iocb to the iocb pool * @phba: Pointer to HBA context object. * @iocbq: Pointer to driver iocb object. * * This function is called with no lock held to release the iocb to * iocb pool.
**/ void
lpfc_sli_release_iocbq(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
{ unsignedlong iflags;
/* * Clean all volatile data fields, preserve iotag and node struct.
*/
spin_lock_irqsave(&phba->hbalock, iflags);
__lpfc_sli_release_iocbq(phba, iocbq);
spin_unlock_irqrestore(&phba->hbalock, iflags);
}
/** * lpfc_sli_cancel_iocbs - Cancel all iocbs from a list. * @phba: Pointer to HBA context object. * @iocblist: List of IOCBs. * @ulpstatus: ULP status in IOCB command field. * @ulpWord4: ULP word-4 in IOCB command field. * * This function is called with a list of IOCBs to cancel. It cancels the IOCB * on the list by invoking the complete callback function associated with the * IOCB with the provided @ulpstatus and @ulpword4 set to the IOCB commond * fields.
**/ void
lpfc_sli_cancel_iocbs(struct lpfc_hba *phba, struct list_head *iocblist,
uint32_t ulpstatus, uint32_t ulpWord4)
{ struct lpfc_iocbq *piocb;
/** * lpfc_sli_iocb_cmd_type - Get the iocb type * @iocb_cmnd: iocb command code. * * This function is called by ring event handler function to get the iocb type. * This function translates the iocb command to an iocb command type used to * decide the final disposition of each completed IOCB. * The function returns * LPFC_UNKNOWN_IOCB if it is an unsupported iocb * LPFC_SOL_IOCB if it is a solicited iocb completion * LPFC_ABORT_IOCB if it is an abort iocb * LPFC_UNSOL_IOCB if it is an unsolicited iocb * * The caller is not required to hold any lock.
**/ static lpfc_iocb_type
lpfc_sli_iocb_cmd_type(uint8_t iocb_cmnd)
{
lpfc_iocb_type type = LPFC_UNKNOWN_IOCB;
if (iocb_cmnd > CMD_MAX_IOCB_CMD) return 0;
switch (iocb_cmnd) { case CMD_XMIT_SEQUENCE_CR: case CMD_XMIT_SEQUENCE_CX: case CMD_XMIT_BCAST_CN: case CMD_XMIT_BCAST_CX: case CMD_ELS_REQUEST_CR: case CMD_ELS_REQUEST_CX: case CMD_CREATE_XRI_CR: case CMD_CREATE_XRI_CX: case CMD_GET_RPI_CN: case CMD_XMIT_ELS_RSP_CX: case CMD_GET_RPI_CR: case CMD_FCP_IWRITE_CR: case CMD_FCP_IWRITE_CX: case CMD_FCP_IREAD_CR: case CMD_FCP_IREAD_CX: case CMD_FCP_ICMND_CR: case CMD_FCP_ICMND_CX: case CMD_FCP_TSEND_CX: case CMD_FCP_TRSP_CX: case CMD_FCP_TRECEIVE_CX: case CMD_FCP_AUTO_TRSP_CX: case CMD_ADAPTER_MSG: case CMD_ADAPTER_DUMP: case CMD_XMIT_SEQUENCE64_CR: case CMD_XMIT_SEQUENCE64_CX: case CMD_XMIT_BCAST64_CN: case CMD_XMIT_BCAST64_CX: case CMD_ELS_REQUEST64_CR: case CMD_ELS_REQUEST64_CX: case CMD_FCP_IWRITE64_CR: case CMD_FCP_IWRITE64_CX: case CMD_FCP_IREAD64_CR: case CMD_FCP_IREAD64_CX: case CMD_FCP_ICMND64_CR: case CMD_FCP_ICMND64_CX: case CMD_FCP_TSEND64_CX: case CMD_FCP_TRSP64_CX: case CMD_FCP_TRECEIVE64_CX: case CMD_GEN_REQUEST64_CR: case CMD_GEN_REQUEST64_CX: case CMD_XMIT_ELS_RSP64_CX: case DSSCMD_IWRITE64_CR: case DSSCMD_IWRITE64_CX: case DSSCMD_IREAD64_CR: case DSSCMD_IREAD64_CX: case CMD_SEND_FRAME:
type = LPFC_SOL_IOCB; break; case CMD_ABORT_XRI_CN: case CMD_ABORT_XRI_CX: case CMD_CLOSE_XRI_CN: case CMD_CLOSE_XRI_CX: case CMD_XRI_ABORTED_CX: case CMD_ABORT_MXRI64_CN: case CMD_XMIT_BLS_RSP64_CX:
type = LPFC_ABORT_IOCB; break; case CMD_RCV_SEQUENCE_CX: case CMD_RCV_ELS_REQ_CX: case CMD_RCV_SEQUENCE64_CX: case CMD_RCV_ELS_REQ64_CX: case CMD_ASYNC_STATUS: case CMD_IOCB_RCV_SEQ64_CX: case CMD_IOCB_RCV_ELS64_CX: case CMD_IOCB_RCV_CONT64_CX: case CMD_IOCB_RET_XRI64_CX:
type = LPFC_UNSOL_IOCB; break; case CMD_IOCB_XMIT_MSEQ64_CR: case CMD_IOCB_XMIT_MSEQ64_CX: case CMD_IOCB_RCV_SEQ_LIST64_CX: case CMD_IOCB_RCV_ELS_LIST64_CX: case CMD_IOCB_CLOSE_EXTENDED_CN: case CMD_IOCB_ABORT_EXTENDED_CN: case CMD_IOCB_RET_HBQE64_CN: case CMD_IOCB_FCP_IBIDIR64_CR: case CMD_IOCB_FCP_IBIDIR64_CX: case CMD_IOCB_FCP_ITASKMGT64_CX: case CMD_IOCB_LOGENTRY_CN: case CMD_IOCB_LOGENTRY_ASYNC_CN:
printk("%s - Unhandled SLI-3 Command x%x\n",
__func__, iocb_cmnd);
type = LPFC_UNKNOWN_IOCB; break; default:
type = LPFC_UNKNOWN_IOCB; break;
}
return type;
}
/** * lpfc_sli_ring_map - Issue config_ring mbox for all rings * @phba: Pointer to HBA context object. * * This function is called from SLI initialization code * to configure every ring of the HBA's SLI interface. The * caller is not required to hold any lock. This function issues * a config_ring mailbox command for each ring. * This function returns zero if successful else returns a negative * error code.
**/ staticint
lpfc_sli_ring_map(struct lpfc_hba *phba)
{ struct lpfc_sli *psli = &phba->sli;
LPFC_MBOXQ_t *pmb;
MAILBOX_t *pmbox; int i, rc, ret = 0;
pmb = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); if (!pmb) return -ENOMEM;
pmbox = &pmb->u.mb;
phba->link_state = LPFC_INIT_MBX_CMDS; for (i = 0; i < psli->num_rings; i++) {
lpfc_config_ring(phba, i, pmb);
rc = lpfc_sli_issue_mbox(phba, pmb, MBX_POLL); if (rc != MBX_SUCCESS) {
lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0446 Adapter failed to init (%d), " "mbxCmd x%x CFG_RING, mbxStatus x%x, " "ring %d\n",
rc, pmbox->mbxCommand,
pmbox->mbxStatus, i);
phba->link_state = LPFC_HBA_ERROR;
ret = -ENXIO; break;
}
}
mempool_free(pmb, phba->mbox_mem_pool); return ret;
}
/** * lpfc_sli_ringtxcmpl_put - Adds new iocb to the txcmplq * @phba: Pointer to HBA context object. * @pring: Pointer to driver SLI ring object. * @piocb: Pointer to the driver iocb object. * * The driver calls this function with the hbalock held for SLI3 ports or * the ring lock held for SLI4 ports. The function adds the * new iocb to txcmplq of the given ring. This function always returns * 0. If this function is called for ELS ring, this function checks if * there is a vport associated with the ELS command. This function also * starts els_tmofunc timer if this is an ELS command.
**/ staticint
lpfc_sli_ringtxcmpl_put(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, struct lpfc_iocbq *piocb)
{
u32 ulp_command = 0;
/** * lpfc_sli_ringtx_get - Get first element of the txq * @phba: Pointer to HBA context object. * @pring: Pointer to driver SLI ring object. * * This function is called with hbalock held to get next * iocb in txq of the given ring. If there is any iocb in * the txq, the function returns first iocb in the list after * removing the iocb from the list, else it returns NULL.
**/ struct lpfc_iocbq *
lpfc_sli_ringtx_get(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
{ struct lpfc_iocbq *cmd_iocb;
/** * lpfc_cmf_sync_cmpl - Process a CMF_SYNC_WQE cmpl * @phba: Pointer to HBA context object. * @cmdiocb: Pointer to driver command iocb object. * @rspiocb: Pointer to driver response iocb object. * * This routine will inform the driver of any BW adjustments we need * to make. These changes will be picked up during the next CMF * timer interrupt. In addition, any BW changes will be logged * with LOG_CGN_MGMT.
**/ staticvoid
lpfc_cmf_sync_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, struct lpfc_iocbq *rspiocb)
{ union lpfc_wqe128 *wqe;
uint32_t status, info; struct lpfc_wcqe_complete *wcqe = &rspiocb->wcqe_cmpl;
uint64_t bw, bwdif, slop;
uint64_t pcent, bwpcent; int asig, afpin, sigcnt, fpincnt; int wsigmax, wfpinmax, cg, tdp; char *s;
/* First check for error */
status = bf_get(lpfc_wcqe_c_status, wcqe); if (status) {
lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT, "6211 CMF_SYNC_WQE Error " "req_tag x%x status x%x hwstatus x%x " "tdatap x%x parm x%x\n",
bf_get(lpfc_wcqe_c_request_tag, wcqe),
bf_get(lpfc_wcqe_c_status, wcqe),
bf_get(lpfc_wcqe_c_hw_status, wcqe),
wcqe->total_data_placed,
wcqe->parameter); goto out;
}
/* Gather congestion information on a successful cmpl */
info = wcqe->parameter;
phba->cmf_active_info = info;
/* See if firmware info count is valid or has changed */ if (info > LPFC_MAX_CMF_INFO || phba->cmf_info_per_interval == info)
info = 0; else
phba->cmf_info_per_interval = info;
/* Get BW requirement from firmware */
bw = (uint64_t)tdp * LPFC_CMF_BLK_SIZE; if (!bw) {
lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT, "6212 CMF_SYNC_WQE x%x: NULL bw\n",
bf_get(lpfc_wcqe_c_request_tag, wcqe)); goto out;
}
/* Gather information needed for logging if a BW change is required */
wqe = &cmdiocb->wqe;
asig = bf_get(cmf_sync_asig, &wqe->cmf_sync);
afpin = bf_get(cmf_sync_afpin, &wqe->cmf_sync);
fpincnt = bf_get(cmf_sync_wfpincnt, &wqe->cmf_sync);
sigcnt = bf_get(cmf_sync_wsigcnt, &wqe->cmf_sync); if (phba->cmf_max_bytes_per_interval != bw ||
(asig || afpin || sigcnt || fpincnt)) { /* Are we increasing or decreasing BW */ if (phba->cmf_max_bytes_per_interval < bw) {
bwdif = bw - phba->cmf_max_bytes_per_interval;
s = "Increase";
} else {
bwdif = phba->cmf_max_bytes_per_interval - bw;
s = "Decrease";
}
/* What is the change percentage */
slop = div_u64(phba->cmf_link_byte_count, 200); /*For rounding*/
pcent = div64_u64(bwdif * 100 + slop,
phba->cmf_link_byte_count);
bwpcent = div64_u64(bw * 100 + slop,
phba->cmf_link_byte_count); /* Because of bytes adjustment due to shorter timer in * lpfc_cmf_timer() the cmf_link_byte_count can be shorter and * may seem like BW is above 100%.
*/ if (bwpcent > 100)
bwpcent = 100;
/* Save BW change to be picked up during next timer interrupt */
phba->cmf_last_sync_bw = bw;
out:
lpfc_sli_release_iocbq(phba, cmdiocb);
}
/** * lpfc_issue_cmf_sync_wqe - Issue a CMF_SYNC_WQE * @phba: Pointer to HBA context object. * @ms: ms to set in WQE interval, 0 means use init op * @total: Total rcv bytes for this interval * * This routine is called every CMF timer interrupt. Its purpose is * to issue a CMF_SYNC_WQE to the firmware to inform it of any events * that may indicate we have congestion (FPINs or Signals). Upon * completion, the firmware will indicate any BW restrictions the * driver may need to take.
**/ int
lpfc_issue_cmf_sync_wqe(struct lpfc_hba *phba, u32 ms, u64 total)
{ union lpfc_wqe128 *wqe; struct lpfc_iocbq *sync_buf; unsignedlong iflags;
u32 ret_val, cgn_sig_freq;
u32 atot, wtot, max;
u8 warn_sync_period = 0;
/* First address any alarm / warning activity */
atot = atomic_xchg(&phba->cgn_sync_alarm_cnt, 0);
wtot = atomic_xchg(&phba->cgn_sync_warn_cnt, 0);
spin_lock_irqsave(&phba->hbalock, iflags);
/* ONLY Managed mode will send the CMF_SYNC_WQE to the HBA */ if (phba->cmf_active_mode != LPFC_CFG_MANAGED ||
phba->link_state < LPFC_LINK_UP) {
ret_val = 0; goto out_unlock;
}
sync_buf = __lpfc_sli_get_iocbq(phba); if (!sync_buf) {
lpfc_printf_log(phba, KERN_ERR, LOG_CGN_MGMT, "6244 No available WQEs for CMF_SYNC_WQE\n");
ret_val = ENOMEM; goto out_unlock;
}
wqe = &sync_buf->wqe;
/* WQEs are reused. Clear stale data and set key fields to zero */
memset(wqe, 0, sizeof(*wqe));
/* If this is the very first CMF_SYNC_WQE, issue an init operation */ if (!ms) {
lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT, "6441 CMF Init %d - CMF_SYNC_WQE\n",
phba->fc_eventTag);
bf_set(cmf_sync_op, &wqe->cmf_sync, 1); /* 1=init */
bf_set(cmf_sync_interval, &wqe->cmf_sync, LPFC_CMF_INTERVAL); goto initpath;
}
/** * lpfc_sli_next_iocb_slot - Get next iocb slot in the ring * @phba: Pointer to HBA context object. * @pring: Pointer to driver SLI ring object. * * This function is called with hbalock held and the caller must post the * iocb without releasing the lock. If the caller releases the lock, * iocb slot returned by the function is not guaranteed to be available. * The function returns pointer to the next available iocb slot if there * is available slot in the ring, else it returns NULL. * If the get index of the ring is ahead of the put index, the function * will post an error attention event to the worker thread to take the * HBA to offline state.
**/ static IOCB_t *
lpfc_sli_next_iocb_slot (struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
{ struct lpfc_pgp *pgp = &phba->port_gp[pring->ringno];
uint32_t max_cmd_idx = pring->sli.sli3.numCiocb;
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.
