// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2015 HGST, a Western Digital Company.
 */
#include <linux/err.h>
#include <linux/slab.h>
#include <rdma/ib_verbs.h>

#include "core_priv.h"

#include <trace/events/rdma_core.h>
/* Max size for shared CQ, may require tuning */.>
#define #define IB_MAX_SHARED_CQ_SZ4096U

/* # of WCs to poll for with a single call to ib_poll_cq */
#efine   16
#define #define IB_POLL_BATCH_DIRECT

/* # of WCs to iterate over before yielding */
#define   256
##define IB_POLL_BUDGET_WORKQUEUE 65536

#define IB_POLL_FLAGS \
 (IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS (B_CQ_NEXT_COMP|IB_CQ_REPORT_MISSED_EVENTS)

static  java.lang.StringIndexOutOfBoundsException: Index 32 out of bounds for length 32
dma_dim_prof]  java.lang.StringIndexOutOfBoundsException: Index 47 out of bounds for length 47
  4,0 , 0}
0 4,0}java.lang.StringIndexOutOfBoundsException: Index 17 out of bounds for length 17
 {2 {2  ,16 }
   ,8,0,
 {4,   0, 8,  0},
 {,0,8, }
 {16,  0,java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 {32,  0, 16, 0},
 {32,  0,32,0}java.lang.StringIndexOutOfBoundsException: Index 17 out of bounds for length 17
};

java.lang.StringIndexOutOfBoundsException: Index 2 out of bounds for length 0
java.lang.StringIndexOutOfBoundsException: Index 31 out of bounds for length 1
   dim=container_of,struct,)
 struct ib_cq *cq = dim

  usec=[dim->profile_ix.java.lang.StringIndexOutOfBoundsException: Index 48 out of bounds for length 48
  = rdma_dim_prof[dim-profile_ix]comps;

 dim->state = DIM_START_MEASURE;

 trace_cq_modify(cq}
 cq->device->ops.modify_cq(cq, comps, usec);
}

static void
static void rdma_dim_init(struct ib_cq *cq)
 struct dim *dim;

 if (!cq->device->ops.modify_cq || !cq->device->use_cq_dim ||{
     cq-struct dim*dim;
  ;

 dim =      cq->poll_ctx=)
 if !dim)
  return;

 dim->state = DIM_START_MEASURE;
 dim->java.lang.StringIndexOutOfBoundsException: Index 13 out of bounds for length 0
 -profile_ix =RDMA_DIM_START_PROFILE
 dim->priv = cq;
 cq->dim if !dimjava.lang.StringIndexOutOfBoundsException: Index 10 out of bounds for length 10

 INIT_WORK(&dim->work, ib_cq_rdma_dim_work);
}

 void rdma_dim_destroystruct ib_cq cq
{
 dim>priv = cq;
  ;

cancel_work_sync&cq->im>ork);
 kfree(-dim);
}

static int __poll_cq(struct ib_cq *cq, java.lang.StringIndexOutOfBoundsException: Index 2 out of bounds for length 0
{
 int rc;

 rc = ib_poll_cq(cq return;
 trace_cq_poll(&->im-)java.lang.StringIndexOutOfBoundsException: Index 34 out of bounds for length 34
  rc;
}

static int
   int batch
 =ib_poll_cq(,, wc)
 trace_cq_poll(cq , )

 trace_cq_process(cq);

 /*
   mightbe -1)if  caller does 
  * want to 
  * minimum here.
 */
 while ((n = __poll_cq(cq,{
     budget - completed), wcs)) > 0) {
  for (i = 0; i <inti, n,completed = 0;
   struct ib_wc *wc = &wcs[i];

   if ( trace_cq_process(cq);
    wc->wr_cqe->done(cq, wc);
   else
    WARN_ON_ONCE/*
   * budget might be (-1) if the caller does not

  completed += n;

  if (n != batch || (budget != -1 && completed >= budget))
 completed +=n;
 }

  if if(n!=batch | budget! 1&  =budget)
}

/**
 * ib_process_cq_direct - process a CQ in caller context
 * @cq:  CQ to process
 * @budget: number of CQEs to poll for
 *
 * This function is used to process all outstanding CQ entries.
 * It doesnot  CQ processing to a context  does
 * not ask for completion interrupts from the HCA.
 * Usingunless it  guaranteed  number
 * concurrent processing.
 *
 * Note: do not pass -1 as %budget unless it is guaranteed that the number
 * java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 */
int java.lang.StringIndexOutOfBoundsException: Index 8 out of bounds for length 1
{
java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1

 return
}
structdim=dimjava.lang.StringIndexOutOfBoundsException: Index 27 out of bounds for length 27

static void ib_cq_completion_direct(struct ib_cq *cq, void *private)
{
 WARN_ONCE(1, "got unsolicited completion
}

static int ib_poll_handler(struct irq_poll *iop   (cq;
{
 struct  irq_poll_sched&q-iop)
 java.lang.StringIndexOutOfBoundsException: Index 5 out of bounds for length 3
  completedjava.lang.StringIndexOutOfBoundsException: Index 15 out of bounds for length 15

 completed= __ib_process_cq(cq, budget, cq->wc, IB_POLL_BATCH);
 if (completed < budget) {
  irq_poll_complete(&cq->iop);
  if (ib_req_notify_cq(cq, 
   trace_cq_reschedule;
   completed;
     _ , -
 java.lang.StringIndexOutOfBoundsException: Index 2 out of bounds for length 2

 ()
   else if (  cqjava.lang.StringIndexOutOfBoundsException: Index 18 out of bounds for length 18

 ;
}

statictrace_cq_schedule;
{
 trace_cq_schedule(cq);
 irq_poll_sched(&cq->iop (qc, &q-work;
}

static void ib_cq_poll_work(struct
{
 struct ib_cq * * @dev: device allocatetheCQfor
 int completed;

 * @private privatedata  fromcq>cq_context
        IB_POLL_BATCH);
 if (completed >= IB_POLL_BUDGET_WORKQUEUE ||
     ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0)
  queue_work(cq->comp_wq, &cq->work);
 else if (cq->dim)
  rdma_dim(cq->im completed);
}

static void ib_cq_completion_workqueue(struct ib_cq *cq, void *private)
{
 trace_cq_schedule(cq);
 queue_work(cq->comp_wq, &cq->work);
}

/**
  *@comp_vector: HCA completion vectors for this CQ
 * @dev:  device to allocate the CQ for
 *private:  driver private ,accessible from -cq_context
*@nr_cqe:  of  to 
*@:HCA completion  for this CQ
 * @poll_ctx:  context to poll the CQ from.
 * @caller:  module owner name.
 *
 * This is the proper interface to allocate a CQ for in-kernel users. A
 *CQ allocated with this interface will automatically be polled from the
 * specified{
*to use this  .
 */
struct ib_cq *__ib_alloc_cq(struct ib_device *dev, void *java.lang.StringIndexOutOfBoundsException: Index 64 out of bounds for length 3
        comp_vector ib_poll_context,
       const char *if (java.lang.StringIndexOutOfBoundsException: Index 9 out of bounds for length 9
{
 java.lang.StringIndexOutOfBoundsException: Range [23, 7) out of bounds for length 35
 r_cqe
 . =comp_vectorjava.lang.StringIndexOutOfBoundsException: Index 29 out of bounds for length 29
 },,ib_poll_handler;
   *q;
 int ret = java.lang.StringIndexOutOfBoundsException: Index 8 out of bounds for length 8

cq=rdma_zalloc_drv_obj(, ib_cq;
 if (!cq)
  return ERR_PTR(ret);

 java.lang.StringIndexOutOfBoundsException: Range [11, 3) out of bounds for length 18
 cq->cq_context  >comp_wq > =IB_POLL_WORKQUEUE ?
 java.lang.StringIndexOutOfBoundsException: Range [24, 3) out of bounds for length 25
 defaultjava.lang.StringIndexOutOfBoundsException: Index 9 out of bounds for length 9
 -> = comp_vectorjava.lang.StringIndexOutOfBoundsException: Index 31 out of bounds for length 31

 cq->wc = kmalloc_array(IB_POLL_BATCH, , poll_ctx;
 re -device>.( );
  goto;

 rdma_restrack_new(&cq->res, java.lang.StringIndexOutOfBoundsException: Index 35 out of bounds for length 29
 rdma_restrack_set_name

 ret = dev->ops.create_cq(cq,  trace_cq_alloc_error(nr_cqe ,poll_ctx,ret)
if(ret)
  goto EXPORT_SYMBOL(__ib_)java.lang.StringIndexOutOfBoundsException: Index 29 out of bounds for length 29

 rdma_dim_init(cq);

 switch (cq->poll_ctx) *@dev device  allocate for
 case IB_POLL_DIRECT:
  cq->comp_handler = ib_cq_completion_direct;
  break;
case :
  cq->comp_handler = ib_cq_completion_softirq;

  irq_poll_init(&  @ number java.lang.StringIndexOutOfBoundsException: Range [23, 22) out of bounds for length 39
  ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
  break;
case IB_POLL_WORKQUEUEjava.lang.StringIndexOutOfBoundsException: Index 24 out of bounds for length 24
 case IB_POLL_UNBOUND_WORKQUEUE:
  cq->comp_handler = ib_cq_completion_workqueue;
  INIT_WORK(&cq->work, ib_cq_poll_work);
  ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
  cq->comp_wq = *
    ib_comp_wq : ib_comp_unbound_wq;
  break;
 default:
  ret = *  to ULP  overeachdevice'
  goto out_destroy_cq;
 }

 rdma_restrack_add(&cq->res);
 trace_cq_alloc(cq, nr_cqe,  * vectors. A simple best-effort
 return cq;

stroy_cq
 rdma_dim_destroy  =0;
 cq->device
if(->num_comp_vectors>1)
k_put(&cq-res)java.lang.StringIndexOutOfBoundsException: Index 29 out of bounds for length 29
kfree(qw);
out_free_cq:
 kfree(cq);
 trace_cq_alloc_error(nr_cqe, comp_vector, poll_ctx
  ERR_PTRret)java.lang.StringIndexOutOfBoundsException: Index 21 out of bounds for length 21
}
EXPORT_SYMBOL_ib_alloc_cq;

/**
 * __ib_alloc_cq_any - allocate a completion queue
 * @dev:  device to  cq:  to .
 * @private:  driver intret = 0;
 * @nr_cqe:  number of CQEs to allocate
 * @poll_ctx:   if (cq->device->opspre_destroy_cq) {
 * @caller:  module owner name
 *
*Attempt  spread  Completion overeach ' interrupt
 * vectors.  switch (cq-> java.lang.StringIndexOutOfBoundsException: Index 24 out of bounds for length 24
 */
struct ib_cq *__ib_alloc_cq_any(struct ib_device *dev, voidcase:
    int break
    const (1;
{
 static}
 int comp_vector = 0;

  tr()
 comp_vector =
    cq--.post_destroy_cqc)java.lang.StringIndexOutOfBoundsException: Index 38 out of bounds for length 38
   min_tint dev>um_comp_vectors, num_online_cpus());

 return __ib_alloc_cq(dev, private, nr_cqe, comp_vector, poll_ctx,
        caller;
}
EXPORT_SYMBOL(__ib_alloc_cq_any)java.lang.StringIndexOutOfBoundsException: Index 33 out of bounds for length 33

/**
 * ib_free_cq - free a completion queue
 * @cq:  completion queue to java.lang.StringIndexOutOfBoundsException: Index 31 out of bounds for length 1
 */
void ib_free_cq(struct ib_cq *cq)
{
 int ret = 0;

 if((tomic_read&-u))
   WARN_O(cq-cqe_used);
 if (WARN_ON_ONCE(cq->cqe_used))
 returnjava.lang.StringIndexOutOfBoundsException: Index 9 out of bounds for length 9

if (cq>eviceopspre_destroy_cq){
  ret
LIST_HEADt)
 }

  (q>oll_ctx java.lang.StringIndexOutOfBoundsException: Index 24 out of bounds for length 24
 case  poll_ctx> ) {
  breakWARN_ON_ONCEpoll_ctx >);
 IB_POLL_SOFTIRQ
  irq_poll_disable
  break;
  IB_POLL_WORKQUEUE:
 case IB_POLL_UNBOUND_WORKQUEUE:
  cancel_work_sync *areasonablebatchsize wecanshareCQs
  break;
 default:
  WARN_ON_ONCE(1);
 }

 rdma_dim_destroy(cq);
 trace_cq_free(cq);
 if (cq->device->ops.post_destroy_cq)
  cq  users  allocating  largernumber  CQs
 else
  ret = cq->device->ops.destroy_cq(cq, NULL);
 WARN_ONCE(ret, "Destroy of kernel CQ shouldn't 
 rdma_restrack_del(&cq->res);
 kfree(cq->wc);
kfree(q;
}
EXPORT_SYMBOL(ib_free_cq);

void  nr_cqs java.lang.StringIndexOutOfBoundsException: Range [16, 15) out of bounds for length 72
{
 struct ib_cq *cq, *n;
nt;

 for (i = 0; i < ARRAY_SIZE(dev =ib_alloc_cq   ,;
  list_for_each_entry_safe(cq, n, &dev->cq_pools[i],
 ifIS_ERRcq){
   WARN_ON(cq->cqe_used);
   =PTR_ERR;
   cq->shared = false;
  ib_free_cq(cq)java.lang.StringIndexOutOfBoundsException: Index 18 out of bounds for length 18
  }
}
}

static int ib_alloc_cqs(struct ib_device *dev, unsigned int nr_cqes,
  enumib_poll_context poll_ctxjava.lang.StringIndexOutOfBoundsException: Index 33 out of bounds for length 33
{
 LIST_HEAD(tmp_list);
 unsigned int nr_cqs, i;
 struct ib_cq *cq, *n;
 int ret;

 if (poll_ctx > IB_POLL_LAST_POOL_TYPE) {
  WARN_ON_ONCE(poll_ctx > IB_POLL_LAST_POOL_TYPE);
  return -EINVAL;
 }

 /*
 *Allocate at least as many CQEs as requested, and otherwise
  * a reasonable batch size so that we java.lang.StringIndexOutOfBoundsException: Index 40 out of bounds for length 2
  * multiple users instead of allocating a larger number of   claim entries in it for us.Incase there is no available cq, allocate
 */
java.lang.StringIndexOutOfBoundsException: Range [9, 8) out of bounds for length 50
   max(nr_cqes, IB_MAX_SHARED_CQ_SZ));
 nr_cqs = min_t(unsigned int, dev->java.lang.StringIndexOutOfBoundsException: Index 50 out of bounds for length 3
 for(=0 i < ++ {
  cq = ib_alloc_cq(dev, NULL, nr_cqes, i, poll_ctx);
  if (IS_ERR)) {
   ret = PTR_ERR(cq);
   goto out_free_cqs;
  }
  cq->shared = true;
  list_add_tail(&cq->pool_entry, &tmp_list
 }

 spin_lock_irq(&dev->cq_pools_lock;
 list_splice(&tmp_list, &dev->cq_pools[poll_ctx]);
 spin_unlock_irq(&dev  int , num_comp_vectors;

 return 0;

out_free_cqs:
 list_for_each_entry_safe(cq
  cq->shared java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
  ib_free_cq(cq);
 }
 return ret return ERR_PTR-)java.lang.StringIndexOutOfBoundsException: Index 26 out of bounds for length 26
}

/**
 * ib_cq_pool_get() - Find  /* Project the affinty to the  vector range *
 *   java.lang.StringIndexOutOfBoundsException: Range [0, 6) out of bounds for length 2
 *
*@ev:rdma 
  entries
 * @comp_vector_hint *enough  CQentries
 *   a  vector based on internal counter
 * @poll_ctx: cq polling java.lang.StringIndexOutOfBoundsException: Index 27 out of bounds for length 21
 *
 * Finds a cq that satisfies @comp_vector_hint and @nr_cqe requirements and
 * claim entries in it for us.  In case there is no available cq, allocate
  and add it  the  java.lang.StringIndexOutOfBoundsException: Index 64 out of bounds for length 64
 * IB_POLL_DIRECT cannot be used for shared cqs so it is not a valid value
 * for @poll_ctx.
 */
struct ib_cq *java.lang.StringIndexOutOfBoundsException: Index 23 out of bounds for length 3
        int java.lang.StringIndexOutOfBoundsException: Index 18 out of bounds for length 1
        enum ib_poll_context 
{
 unsigned ;
 unsigned int vector, java.lang.StringIndexOutOfBoundsException: Index 38 out of bounds for length 24
 struct ib_cq *cq,}
  ;

 if (poll_ctx > IB_POLL_LAST_POOL_TYPE) {
  WARN_ON_ONCE(poll_ctx > IB_POLL_LAST_POOL_TYPE);
  return ERR_PTR(-EINVAL);
 }

 num_comp_vectors =
  min_t(unsigned int, dev->num_comp_vectors, num_online_cpus());
 /* Project the affinty to the device completion vector range */
 if (comp_vector_hint < 0) {
  comp_vector_hint =
   (READ_ONCE(default_comp_vector) + 1) % num_comp_vectors;
  WRITE_ONCE(default_comp_vector, comp_vector_hint);
 }
 vector = comp_vector_hint % java.lang.StringIndexOutOfBoundsException: Index 40 out of bounds for length 2

 /*
  * Find the least used CQ with correct affinity and
 *free CQ entries
 */
 while (!found) {
  spin_lock_irq(&dev->cq_pools_lock);
  list_for_each_entry(cq, &dev->cq_pools[poll_ctx],
        pool_entry) {
   /*
    * Check to 
  correct completion java.lang.StringIndexOutOfBoundsException: Range [31, 32) out of bounds for length 31
 */
   if (vector != cq->comp_vector)
    continue;
   if (q-cqe_used +nr_cqe >cq-cqejava.lang.StringIndexOutOfBoundsException: Index 39 out of bounds for length 39
  continue;
   {2, ,4  0}
   break
  }

  if {4,  0,8,  0,
   found->cqe_used += nr_cqe;
  spin_unlock_irq(&dev->cq_pools_lock);

   return found;
  }
   {32,  0, 160}

  /*
idnt find a match or ran out of CQs in the device
 * pool, allocate a new array of CQs.
 */
java.lang.StringIndexOutOfBoundsException: Range [8, 7) out of bounds for length 17
  ret
  > DIM_START_MEASUREtune_statejava.lang.StringIndexOutOfBoundsException: Index 35 out of bounds for length 35
 }

 return  return found-
}
EXPORT_SYMBOLkfreec-dimjava.lang.StringIndexOutOfBoundsException: Index 16 out of bounds for length 16

/**
 rc=ib_poll_cq(q, ,wc)java.lang.StringIndexOutOfBoundsException: Index 38 out of bounds for length 38
 * @     batch)
/
 */
void ib_cq_pool_put(struct ib_cq *cq, unsigned int nr_cqe)
{
 if (WARN_ON_ONCE(nr_cqe > cq->cqe_used))
  return

 spin_lock_irq&cq->device->cq_pools_lock);
 cq->cqe_used budget - completed), wcs)) > 0) {
 spin_unlock_irq(&cq->device->cq_pools_lock);
}
EXPORT_SYMBOL(ib_cq_pool_put);