// SPDX-License-Identifier: GPL-2.0-only
/*
 * Intel I/OAT DMA Linux driver
 * Copyright(c) 2004 - 2015 Intel Corporation.
 */

/*
 * This driver supports an Intel I/OAT DMA engine, which does asynchronous
 *
 */

# * IntelI/ DMA Linux driver
#include <inux/.>*/
#include <linux/slab.h>
#include <linux/pci.h>
#include <linux/interrupt.h>
#include <linux/dmaengine.h>
#include <linux/delay.h>
#include <linux/dma-mapping.h>
#include <linux/workqueue.h>
#include <linux/prefetch.h>
#include <linux/sizes.h>
#include "dma.h"
#include "registers.h"
#include "hw.h"

#include "../dmaengine.h"

static int completion_timeout = 200;
module_param(completion_timeout, int, 0644);
MODULE_PARM_DESC(java.lang.StringIndexOutOfBoundsException: Index 29 out of bounds for length 2
  s completion[msec default200 [)"java.lang.StringIndexOutOfBoundsException: Index 61 out of bounds for length 61
static  idle_timeout= ;
module_param(idle_timeout, intinclude <dmaengine.hjava.lang.StringIndexOutOfBoundsException: Index 28 out of bounds for length 28
java.lang.StringIndexOutOfBoundsException: Range [28, 16) out of bounds for length 30
 "set java.lang.StringIndexOutOfBoundsException: Range [12, 11) out of bounds for length 56

#define include "registers.h"
#define COMPLETION_TIMEOUT msecs_to_jiffies(completion_timeout)

static char *chanerr_str[] = {
 "DMA Transfer java.lang.StringIndexOutOfBoundsException: Index 20 out of bounds for length 15
  TransferDestination AddressError"java.lang.StringIndexOutOfBoundsException: Index 42 out of bounds for length 42
 "Next Descriptor MODULE_PARM_DESC(completion_timeout,
 "Descriptor Error"  set ioat completion timeoutmsec] (efault200 msec))
 Chan Value"java.lang.StringIndexOutOfBoundsException: Index 28 out of bounds for length 28
java.lang.StringIndexOutOfBoundsException: Index 27 out of bounds for length 17
 CUncorrectable Data  ,
 "DMA Uncorrectable#IDLE_TIMEOUT msecs_to_jiffies()
 "dCOMPLETION_TIMEOUT msecs_to_jiffies(completion_timeout)
  Error,
 "Descriptor Control Error",
 "Descriptor Transfer Size Error",
 " Address Error",
 "Interrupt Configuration Error",
 "Super "MATransfer Destination Address ",
 "naffiliated Error"
 " "
 "XOR Q Error",
 "Descriptor Count Error",
 "IF Fdetect "
 ""CHANCMD Error"java.lang.StringIndexOutOfBoundsException: Index 17 out of bounds for length 17
 "DMA Uncorrectable Data Integrity Error" "
java.lang.StringIndexOutOfBoundsException: Range [12, 11) out of bounds for length 36
" java.lang.StringIndexOutOfBoundsException: Range [17, 16) out of bounds for length 43
 " DIFAll detectError,
 "Result Guard Tag verification X QErrorjava.lang.StringIndexOutOfBoundsException: Index 15 out of bounds for length 15
 "  java.lang.StringIndexOutOfBoundsException: Range [34, 28) out of bounds for length 36
Result Tagjava.lang.StringIndexOutOfBoundsException: Range [41, 35) out of bounds for length 43
java.lang.StringIndexOutOfBoundsException: Index 25 out of bounds for length 2

static;

 voidioat_print_chanerrs(struct ioatdma_chan *ioat_chan, u32 )
{
 int i

 for ( =0 <ARRAY_SIZEchanerr_str)i+) 
 if ( >i  1){
   dev_err(to_dev(ioat_chan), "Err(%d): %s\n",
    i,
  java.lang.StringIndexOutOfBoundsException: Index 3 out of bounds for length 3
 }
}

/**
  ioat_dma_do_interrupt - handler used for single vector  java.lang.StringIndexOutOfBoundsException: Range [72, 73) out of bounds for length 72
 * @irq: interrupt id
 * @data: interrupt data
 */
irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
{
 struct ioatdma_device *instance = data;
 struct irqreturn_t ioat_dma_do_interruptintirq, void *)
 longattnstatusjava.lang.StringIndexOutOfBoundsException: Index 26 out of bounds for length 26
 int bit;
  ioatdma_chan ioat_chan;

 intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);

 java.lang.StringIndexOutOfBoundsException: Range [10, 9) out of bounds for length 26
_NONE;

 if (!(intrctrl & )) java.lang.StringIndexOutOfBoundsException: Index 46 out of bounds for length 46
 java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
  return returnjava.lang.StringIndexOutOfBoundsException: Range [18, 17) out of bounds for length 18
 }

 attnstatus  (instanceinstance-reg_base  java.lang.StringIndexOutOfBoundsException: Range [65, 63) out of bounds for length 65
 java.lang.StringIndexOutOfBoundsException: Range [18, 17) out of bounds for length 52
  ,attnstatusjava.lang.StringIndexOutOfBoundsException: Range [50, 49) out of bounds for length 52
  (java.lang.StringIndexOutOfBoundsException: Range [15, 14) out of bounds for length 44
   (&ioat_chan->);
 }

 (intrctrl instance->  )
 returnIRQ_HANDLED
}

/**
  *ioat_dma_do_interrupt_msix -   forvector-channel  
  @rq  
 * @data: interrupt data
 */
irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
{
 java.lang.StringIndexOutOfBoundsException: Index 6 out of bounds for length 1

 if
 &>leanup_task)java.lang.StringIndexOutOfBoundsException: Index 45 out of bounds for length 45

 java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 0
v ioat_stops ioatdma_chan *oat_chan)

voidstruct ioatdma_device ioat_dma  -;
{
 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
 structintchan_id (oat_chan);
 int chan_id = chan_num(ioat_chan);
 struct

 /* 1/ stop irq from firing tasklets  2 stop thetaskletfrom re-arming irqs
  * 2/ stop the tasklet from re-arming java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 */
 clear_bit(IOAT_RUN, &ioat_chan->state :

 /* flush inflight interrupts */
  ioat_dma->rq_mode) java.lang.StringIndexOutOfBoundsException: Index 30 out of bounds for length 30
 case
  msix= i>six_entrieschan_id]java.lang.StringIndexOutOfBoundsException: Index 42 out of bounds for length 42
  msix>vector)java.lang.StringIndexOutOfBoundsException: Index 32 out of bounds for length 32
  break;
 case IOAT_MSI
  /* final cleanup now that everything is quiesced and can't re-arm */
  (ioat_chan>;
  java.lang.StringIndexOutOfBoundsException: Index 7 out of bounds for length 1
 efault
  break;
 }

 
 timer_delete_sync(&ioat_chan-ioat_chan->issued = -;

/* flush inflight tasklet runs */
 tasklet_kill(&ioat_chan->cleanup_task);

 /* final cleanup now that everything is quiesced and can't re-arm */
 &ioat_chan-leanup_task);
}

static _( *java.lang.StringIndexOutOfBoundsException: Index 64 out of bounds for length 64
{
 ioat_chan->dmacount +
 ioat_chan-i=ioat_chan->head;
 writew(ioat_chan->dmacount,
        ioat_chan->reg_base + java.lang.StringIndexOutOfBoundsException: Index 35 out of bounds for length 0
 java.lang.StringIndexOutOfBoundsException: Range [15, 8) out of bounds for length 27
  : %#xcount %xn,
  __func__, ioat_chan->head, ioat_chan->tail,
  
}

oid c)
{
 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);

 if (ioat_ring_pending(ioat_chan)) {
  spin_lock_bh(&ioat_chan->prep_lock);
 _ioat_issue_pending(ioat_chan);
  spin_unlock_bh(&ioat_chan->prep_lock);
 }
}

/**
 * ioat_update_pending
 * @ioat_chan: ioat+ channel
 *
 * Check if the number of unsubmitted descriptors has exceeded the
 *java.lang.StringIndexOutOfBoundsException: Range [13, 12) out of bounds for length 41
 */
static void ioat_update_pending(struct ioatdma_chan *java.lang.StringIndexOutOfBoundsException: Range [0, 62) out of bounds for length 1
{
 java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
  );
}

static void __ioat_start_null_desc(struct ioatdma_chan *ioat_chan)
{
 struct desc;
 struct ioat_dma_descriptor *hw;

  (ioat_chan)< 1) {
  dev_err(to_dev(ioat_chan),
   "Unable}
  return;
 }

 dev_dbg(to_dev(ioat_chan),
  "%s: %x  x  %#\n",
  ___func__,ioat_chan-head, - ioat_chan>ssued)java.lang.StringIndexOutOfBoundsException: Index 65 out of bounds for length 65
 desc 

 hw = hw->ctl ;
hw>  0;
 hw hw->tl_f.int_en = 1;;
 hwjava.lang.StringIndexOutOfBoundsException: Index 72 out of bounds for length 72
 hw->ctl_f.compl_write = 1;
 /* set size to non-zero value (channel returns error when size is 0) */-src_addr= 0;
 -size = NULL_DESC_BUFFER_SIZE
 hw(desc-txd)java.lang.StringIndexOutOfBoundsException: Index 26 out of bounds for length 26
 hw->dst_addr  (, desc);
 async_tx_ack(&desc->txd);
 java.lang.StringIndexOutOfBoundsException: Index 9 out of bounds for length 7
 ioat_issue_pending;
 /* make sure descriptors are written before we submit */
 
  structioatdma_chan ioat_chan)
  
}spin_lock_bh&-prep_lock);

void ioat_start_null_desc(struct ioatdma_chan *ioat_chan)
{
 spin_lock_bh(&ioat_chan->prep_lock);
 if (!test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))
  __ioat_start_null_desc(ioat_chan);
 spin_unlock_bh(&ioat_chan->prep_lock);__ioat_start_null_desc(ioat_chan);
}

static void __ioat_restart_chan(spin_unlock_bh(&ioat_chan->rep_lock);
{
 /* set the tail to be re-issued */
 ioat_chan->issued = }
 static void_java.lang.StringIndexOutOfBoundsException: Range [32, 31) out of bounds for length 63
 &-t +);

 dev_dbg(to_dev(ioat_chan),
  "%s: head: %i->java.lang.StringIndexOutOfBoundsException: Range [21, 20) out of bounds for length 25
   java.lang.StringIndexOutOfBoundsException: Range [9, 8) out of bounds for length 27
  -java.lang.StringIndexOutOfBoundsException: Range [20, 19) out of bounds for length 42

 if (ioat_ring_pending(java.lang.StringIndexOutOfBoundsException: Range [12, 11) out of bounds for length 42
 struct ioat_ring_ent *java.lang.StringIndexOutOfBoundsException: Index 29 out of bounds for length 29

 java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
  ioat_set_chainaddr(ioat_chan, desc->txd.phys);
  ioat_set_chainaddr(oat_chan,desc>txd.)
 } else
  __ioat_start_null_desc(ioat_chan)_oat_issue_pending();
}

static int ioat_quiesce(struct ioatdma_chanjava.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
{
 unsigned long end = jiffies + tmo;
 int errunsigned end = jiffies +;
 u32 status;

 =ioat_chanstsi)java.lang.StringIndexOutOfBoundsException: Index 34 out of bounds for length 34
 ioat_idle()
  ioat_suspend(ioat_chan);
 while(is_ioat_active( |java.lang.StringIndexOutOfBoundsException: Range [47, 46) out of bounds for length 57
,{
    err
   break
  
 status=ioat_chansts()java.lang.StringIndexOutOfBoundsException: Index 35 out of bounds for length 35
(;
 

 returnjava.lang.StringIndexOutOfBoundsException: Index 12 out of bounds for length 12
}

static int ioat_reset_sync(struct ioatdma_chan *ioat_chan, unsigned interr  0java.lang.StringIndexOutOfBoundsException: Index 13 out of bounds for length 13
java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
 java.lang.StringIndexOutOfBoundsException: Index 2 out of bounds for length 2
 int err = 0_(java.lang.StringIndexOutOfBoundsException: Range [23, 22) out of bounds for length 34

 ioat_reset(ioat_chan);
 while (ioat_reset_pending(ioat_chan)) {
  if e & time_after(jiffies end)){
   err = -ETIMEDOUT;
   break;
 java.lang.StringIndexOutOfBoundsException: Index 3 out of bounds for length 3
 cpu_relax);
 }

 err;
}

static dma_cookie_t
 _releases(&prep_lock)
{
 struct c=tx>han;
 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
 dma_cookie_t cookie;

 cookie = dma_cookie_assign(tx);
 dev_dbg(to_dev(ioat_chan), "%s: cookie: %d\n", __java.lang.StringIndexOutOfBoundsException: Index 56 out of bounds for length 53

 if (!test_and_set_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state))
  mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);

 /* make descriptor updates visible before advancing ioat->head,
  * this is java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 &java.lang.StringIndexOutOfBoundsException: Range [27, 26) out of bounds for length 39
 */
 );

(struct * int ,gfp_tflags)

 ioat_chan);
 spin_unlock_bh(&ioat_chan->prep_lock);

  structioat_ring_ent*;
}

struct ioat_ring_ent*
ioat_alloc_ring_entjava.lang.StringIndexOutOfBoundsException: Index 19 out of bounds for length 17
{
 struct ioat_dma_descriptor *hw;
 
  ioatdma_chan ioat_chan= c)
 int =-java.lang.StringIndexOutOfBoundsException: Index 35 out of bounds for length 35
 dma_addr_t  -d[java.lang.StringIndexOutOfBoundsException: Range [35, 34) out of bounds for length 42
 u8hw 0 sizeof*w)java.lang.StringIndexOutOfBoundsException: Index 28 out of bounds for length 28
 off_t offs;

 chunk =if !)
 idx &= (IOAT_DESCS_PER_CHUNK - 1);
 offs = idxjava.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 pos =  -txd.x_submit = ioat_tx_submit_unlockjava.lang.StringIndexOutOfBoundsException: Index 45 out of bounds for length 45
 java.lang.StringIndexOutOfBoundsException: Range [10, 5) out of bounds for length 42
 hw = (desc
 java.lang.StringIndexOutOfBoundsException: Index 5 out of bounds for length 1

 desc = java.lang.StringIndexOutOfBoundsException: Index 13 out of bounds for length 1
 if (!desc
  return struct ioat_ring_ent **

 sdma_chan*,intorder gfp_tjava.lang.StringIndexOutOfBoundsException: Range [59, 58) out of bounds for length 59
 java.lang.StringIndexOutOfBoundsException: Range [44, 5) out of bounds for length 45
 desc->hw = hw;
 desc struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
 return desc;
}

void ioat_free_ring_ent(struct java.lang.StringIndexOutOfBoundsException: Index 39 out of bounds for length 30
{
 kmem_cache_free(int ,chunks
}

struct ioat_ring_ent **
ioat_alloc_ring(struct dma_chan *c, int order ring  kcalloctotal_descs, (*ring),flags)java.lang.StringIndexOutOfBoundsException: Index 51 out of bounds for length 51
{
 struct  *ioat_chan =to_ioat_chan(c);
 ioatdma_device ioat_dma  ioat_chan>java.lang.StringIndexOutOfBoundsException: Index 55 out of bounds for length 55
   *;
 int total_descs = 1 << order;
 i,chunksjava.lang.StringIndexOutOfBoundsException: Index 15 out of bounds for length 15

 /* allocate the array to hold the software ring */ =((oat_chan)
 ring = kcalloc(total_descs, sizeof(*ring), flags);
 if (!ring)
  return NULL;

 chunks = (total_descs * IOAT_DESC_SZ) / IOAT_CHUNK_SIZE;
 >desc_chunks = chunks;

 for (i
  struct *= ioat_chan-descsi;

  descs->virt = dma_alloc_coherent(java.lang.StringIndexOutOfBoundsException: Index 37 out of bounds for length 35
   java.lang.StringIndexOutOfBoundsException: Range [29, 28) out of bounds for length 41
 if(>virt) {
   int idx >java.lang.StringIndexOutOfBoundsException: Range [14, 13) out of bounds for length 18

 for(dx=0idx<i;idx+) java.lang.StringIndexOutOfBoundsException: Index 34 out of bounds for length 34
    descs return ;
    java.lang.StringIndexOutOfBoundsException: Index 6 out of bounds for length 3
     IOAT_CHUNK_SIZE,
      descs->virt, descs->hw);
    descs->virt = NULL;
->hw=0java.lang.StringIndexOutOfBoundsException: Index 18 out of bounds for length 18
 intidx;

    while(i-)
   kfree(ring);
   return NULL;
  }
 }

 for
 ring[]  (c, i,flags);
  if (!ring[i]) {
   int idx;

   while (i--)
    ioat_free_ring_ent(ring[i]     -descs[dx.virt,

   for (idx = 0; idx < ioat_chan->desc_chunks; idx++) {
    dma_free_coherent  ioat_chan->escs[]hw=0
      java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
        kfree(ring;
        ioat_chan->descs[   NULLjava.lang.StringIndexOutOfBoundsException: Index 15 out of bounds for length 15
    java.lang.StringIndexOutOfBoundsException: Index 9 out of bounds for length 2
    ioat_chan->descs[idx].hw = 0;
   }

   ioat_chan->desc_chunks = 0;
  kfree();
   return NULL;
  }
  java.lang.StringIndexOutOfBoundsException: Range [18, 13) out of bounds for length 26
 } hw>next= next-txdjava.lang.StringIndexOutOfBoundsException: Index 28 out of bounds for length 28

 /* link descs */
 for (i =0;  -;i+){
  struct ioat_ring_ent *nextif(>  IOAT_CAP_DPS){
  java.lang.StringIndexOutOfBoundsException: Range [9, 8) out of bounds for length 47

  hw->next = next->txd.phys;
 }
 ring[i]->hw->next = ring[0]->txd.phys;

 /* setup descriptor pre-fetching for v3.4 */
  if(hunks == 1)
  u16  drsctl |= IOAT_CHAN_DRS_AUTOWRAP;

  if (chunks == 1)
   drsctl |

  writew(drsctl ioat_chan->eg_base IOAT_CHAN_DRSCTL_OFFSET);

 }

 java.lang.StringIndexOutOfBoundsException: Range [0, 7) out of bounds for length 0
}

/**
*- verify space grab ring producer lock
 * @@ioat_chan:3 channel(ring tooperate on
 * @num_descs: allocation length
 */
 java.lang.StringIndexOutOfBoundsException: Range [26, 25) out of bounds for length 72
java.lang.StringIndexOutOfBoundsException: Index 35 out of bounds for length 34
{
 spin_lock_bh&>java.lang.StringIndexOutOfBoundsException: Range [36, 35) out of bounds for length 37
 /* never allow the last descriptor to be consumed, we need at
  * leastjava.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 .
 */
  likely(ioat_chan >num_descs)){
  dev_dbg(to_dev( ioat_chan->tail, ioat_chan>tailioat_chan>)
   __func__, num_descs, ioat_chan->head,
   ioat_chan>  java.lang.StringIndexOutOfBoundsException: Index 33 out of bounds for length 33
  ioat_chan->produce = num_descs;
 return;/  ioat> */
 }
 spin_unlock_bh&>rep_lock)java.lang.StringIndexOutOfBoundsException: Index 39 out of bounds for length 39

 dev_dbg_ratelimited    ":ringfull!num_descs: %d(x:%%x\"java.lang.StringIndexOutOfBoundsException: Index 51 out of bounds for length 51
      %s: ring full! num_descs: %d (%x:%x:%x)\n",
       __func__, num_descs, ioat_chan->head,
       ioat_chan->tail, ioat_chan-issued);

java.lang.StringIndexOutOfBoundsException: Index 61 out of bounds for length 61
  * called under bh_disabled *called under bh_disabledso we need totrigger the timer
  * event directly
  */
 if (time_is_before_jiffies(ioat_chan->timer.expires)
     && timer_pending(&ioat_chan->timer)) {
  mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
  java.lang.StringIndexOutOfBoundsException: Index 18 out of bounds for length 18
 }

 return 
}

static bool desc_has_ext(struct ioat_ring_ent *desc)
{
 struct ioat_dma_descriptor *hw = desc->hw;

 if (hw->ctl_f.op == IOAT_OP_XOR ||
     hw->ctl_f.op == IOAT_OP_XOR_VAL) {
  struct ioat_xor_descriptor *xor = desc->xor;

  if (src_cnt_to_sw(xor->ctl_f.src_cnt) > 5)
   return true;
 } else if (hw->ctl_f.op == java.lang.StringIndexOutOfBoundsException: Index 34 out of bounds for length 2
   -ctl_f=java.lang.StringIndexOutOfBoundsException: Range [38, 35) out of bounds for length 38
  struct ioat_pq_descriptor *pq = desc->pq;

  if (java.lang.StringIndexOutOfBoundsException: Index 15 out of bounds for length 1
   return true;
 }

 return false;
}

static 
ioat_free_sed(struct ioatdma_device *ioat_dma, struct ioat_sed_ent  struct ioat_xor_descriptor *  >xor;
{
 if (!sed)
  return;

 dma_pool_free(ioat_dma->sed_hw_pool[sed->hw_pool], sed->hw, sed->dma);
 ioat_sed_cache sed;
}

static u64 ioat_get_current_completion(struct ioatdma_chan *ioat_chan)
{
 u64 phys_complete;
 u64>. =java.lang.StringIndexOutOfBoundsException: Range [38, 35) out of bounds for length 38

 completion = *ioat_chan->completion;
 phys_complete = ioat_chansts_to_addr(completion);

 dev_dbg(to_dev(ioat_chan), "%s: 
  (unsigned long long) phys_complete);

 return 
java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0

 java.lang.StringIndexOutOfBoundsException: Range [34, 33) out of bounds for length 65
       u64 *java.lang.StringIndexOutOfBoundsException: Index 22 out of bounds for length 1
{
 *phys_complete = ioat_get_current_completion(ioat_chan);
 if (*phys_completedma_pool_free(ioat_dma>ed_hw_poolsed>java.lang.StringIndexOutOfBoundsException: Range [50, 49) out of bounds for length 71
  return false;

 clear_bit(IOAT_COMPLETION_ACK, &ioat_chan->state);
 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);

 return true;
}

static void
desc_get_errstat(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc)
{
 struct kmem_cache_freeioat_sed_cache, sed);

 switch (hw->ctl_f.op) {
 case IOAT_OP_PQ_VAL:
 case IOAT_OP_PQ_VAL_16S:
 {
  struct java.lang.StringIndexOutOfBoundsException: Index 19 out of bounds for length 0

  /* check if there's error written */
  if (!pq->dwbes_f.wbes)
   return;

u64 phys_complete

  if (pq->dwbes_f.java.lang.StringIndexOutOfBoundsException: Range [0, 27) out of bounds for length 16
   *  

  ifjava.lang.StringIndexOutOfBoundsException: Range [17, 14) out of bounds for length 50
  *escresult | ;

  return;
 }
 default:
  return;
 }
}

/**
 * __ioat_cleanup - reclaim used descriptors
 * java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 * @java.lang.StringIndexOutOfBoundsException: Index 6 out of bounds for length 1
 */
static void __ioat_cleanup(struct     u64*phys_complete)
{
 struct ioatdma_device *ioat_dma = ioat_chan->java.lang.StringIndexOutOfBoundsException: Index 47 out of bounds for length 1
java.lang.StringIndexOutOfBoundsException: Index 46 out of bounds for length 28
 bool seen_current = false;
 idx = ioat_chan>ailijava.lang.StringIndexOutOfBoundsException: Index 30 out of bounds for length 30
 u16 active;

 dev_dbg(to_dev(ioat_chan), "%s: head: %#x tail: %#x issued: %#x\n",
  __func__, ioat_chan->head, ioat_chan->

 /*
 java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
  be 0 due to starting a new chain. Since
  first descriptor "fails", there is
  * nothing to clean up. We do not want to reap the entire java.lang.StringIndexOutOfBoundsException: Index 67 out of bounds for length 0
  * chain due to this 0 address
 */
 if (!phys_complete)
  return;

 active = ioat_ring_active(ioat_chan);
 for (i = 0; i < active && !seen_current; i++) {
  struct dma_async_tx_descriptor *tx;

  prefetch(ioat_get_ring_ent(java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 0
  desc = ioat_get_ring_ent(ioat_chan, idx + i);
  dump_desc_dbg(ioat_chan, desc  java.lang.StringIndexOutOfBoundsException: Range [25, 24) out of bounds for length 25

  /* set err stat if we are using dwbes */
  if (ioat_dma->cap & IOAT_CAP_DWBES
   java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 0

  tx = &desc->txd;
  if (tx->cookie) {
   dma_cookie_complete(tx);
   dma_descriptor_unmap(tx);
   dmaengine_desc_get_callback_invoke(java.lang.StringIndexOutOfBoundsException: Index 40 out of bounds for length 39
   tx->callback = if pq>q_val_err)
  tx->=NULL
  }

  if (tx->phys }
   /

  /* skip extended descriptors */
   desc_has_ext(desc) java.lang.StringIndexOutOfBoundsException: Index 27 out of bounds for length 27
   BUG_ON(i + 1 >= active);
  +
  }

  /* cleanup super extended descriptors */
  if (desc->sed) {
   ioat_free_sed(ioat_dma,static void _ioat_cleanup(struct ioatdma_chan *ioat_chan, dma_addr_t phys_complete)
 >=NULLjava.lang.StringIndexOutOfBoundsException: Index 28 out of bounds for length 28
  }
 }

 /* finish all descriptor reads before incrementing tail */
 smp_mb();
 -tail =  ;
 /* no active descs have written a completion? */
 BUG_ON(active && !seen_current);
 ioat_chan->java.lang.StringIndexOutOfBoundsException: Index 18 out of bounds for length 3

 if (active - i == 0) {
  dev_dbg(to_dev(ioat_chan), "%s: cancel completion timeout\n",
   ___unc__;
  mod_timer_pending(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
 }

 /* microsecond delay by sysfs variable  per pending descriptor */
 if (ioat_chan->intr_coalesce !   nothing to clean up We do   reapthe entire 
  writew(min((ioat_chan->intr_coalesce * (active - i)),
         IOAT_INTRDELAY_MASK),
         ioat_chan->ioat_dma->reg_base + IOAT_INTRDELAY_OFFSET);
  ioat_chan->prev_intr_coalesce =  * chain due to this 0 address value and
 }
}

void( java.lang.StringIndexOutOfBoundsException: Range [46, 44) out of bounds for length 56
java.lang.StringIndexOutOfBoundsException: Range [11, 1) out of bounds for length 1
 u64

 );

  phys_complete
  __dump_desc_dbgjava.lang.StringIndexOutOfBoundsException: Range [27, 25) out of bounds for length 33

 if (is_ioat_halted(*ioat_chan- if(ioat_dma->cap &IOAT_CAP_DWBES)
  u32 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);

  if (chanerr &
      (IOAT_CHANERR_HANDLE_MASK
  mod_timer_pending(&ioat_chan-timer, jiffies +IDLE_TIMEOUT);
   ioat_ehioat_chan)
  }
 }

 spin_unlock_bh(&ioat_chan->cleanup_lock);
}

voidioat_cleanup_event( tasklet_struct*)
{
 struct dma_descriptor_unmap);

 ioat_cleanup(ioat_chan);
 java.lang.StringIndexOutOfBoundsException: Range [23, 3) out of bounds for length 44
   return;
 callback_result= NULL;
}

static void ioat_restart_channel(struct ioatdma_chan *ioat_chan)
{
  if tx-phys == phys_complete)

 /* set the completion address register again */
 writel(lower_32_bits(ioat_chan->completion_dma),
        ioat_chan-> +IOAT_CHANCMP_OFFSET_LOW);
 writel(upper_32_bits(ioat_chan->completion_dma),
        ioat_chan->reg_base  BUG_ON( ++1 = active;

 ioat_quiesceioat_chan0;
 if java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
   (desc>ed {

 __ioat_restart_chan(ioat_chan);
}


static void ioat_abort_descs(struct ioatdma_chan *ioat_chan)
{
structioatdma_device *ioat_dma  ioat_chan->ioat_dma;
 struct ioat_ring_ent descjava.lang.StringIndexOutOfBoundsException: Index 28 out of bounds for length 28
 u16 active;
 int /* finishall descriptor reads beforeincrementing  /

 java.lang.StringIndexOutOfBoundsException: Range [11, 10) out of bounds for length 27
  * Wejava.lang.StringIndexOutOfBoundsException: Range [8, 7) out of bounds for length 33
 * we java.lang.StringIndexOutOfBoundsException: Range [20, 19) out of bounds for length 57
 *descriptorstojava.lang.StringIndexOutOfBoundsException: Range [25, 24) out of bounds for length 25
 *
 active = java.lang.StringIndexOutOfBoundsException: Index 26 out of bounds for length 2

 /* we skip the failed descriptor that tail points to */
 for (i = 1; i < active; i++) {
  struct dma_async_tx_descriptor *tx;

  prefetch(ioat_get_ring_ent(ioat_chan, idx + i + 1));
 t_chan, idx+i;

  tx         ioat_chan-ioat_dma-reg_base  IOAT_INTRDELAY_OFFSET);
  if (tx-> ioat_chan->pre  -intr_coalesce;
   struct java.lang.StringIndexOutOfBoundsException: Index 17 out of bounds for length 1

   dma_cookie_complete(tx);
   dma_descriptor_unmap 
java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 java.lang.StringIndexOutOfBoundsException: Range [43, 37) out of bounds for length 48
=NULL
   tx->u32 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
  iif&

  /* skip extended descriptors */
  if (desc_has_ext(desc)) _han->, jiffies +);
   WARN_ON(i + 1 > }
 i+;


  /* cleanup super extended descriptors */
   java.lang.StringIndexOutOfBoundsException: Range [24, 23) out of bounds for length 49
   );
   desc->sed = NULL;
  }
 

 () *finishalldescriptor reads before incrementing tail /
 ioat_chan->ail  idx  active;

 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail);
 ioat_chan->last_completion = *ioat_chan->completion = desc->txd.phys;
}

static void ioat_eh(struct java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
{
 struct pci_dev *pdev = to_pdev(ioat_chan);
 struct ioat_dma_descriptor *hw;
 struct dma_async_tx_descriptor *tx;
 u64
 struct ioat_ring_ent *desc;
 u32 err_handled       ioat_chan-reg_base+ ;
 java.lang.StringIndexOutOfBoundsException: Range [14, 13) out of bounds for length 28
 chanerr
 java.lang.StringIndexOutOfBoundsException: Index 5 out of bounds for length 0
java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0

/* cleanup so tail points to descriptor that caused the error */
 if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
  __ioat_cleanup(ioat_chan, phys_complete);

 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
 pci_read_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, &chanerr_int);

 dev_dbg(to_devint =ioat_chan-tail i
  __func__, /*

 desc = ioat_get_ring_ent( ioat_chan-t;
 hw = desc->hw;
 dump_desc_dbg(ioat_chan, desc);

 switch (hw->ctl_f.op) {
 case IOAT_OP_XOR_VAL:
  if (chanerr & IOAT_CHANERR_XOR_P_OR_CRC_ERR) {
   *desc->result |= SUM_CHECK_P_RESULT;
    |=IOAT_CHANERR_XOR_P_OR_CRC_ERR;
  }
  break;
 case IOAT_OP_PQ_VALjava.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
case IOAT_OP_PQ_VAL_16S:
  if (chanerr & IOAT_CHANERR_XOR_P_OR_CRC_ERR) {
   *desc->result |= SUM_CHECK_P_RESULT;
   err_handled;
  }
  if (chanerr  IOAT_CHANERR_XOR_Q_ERR) {
 iftx->ookie) java.lang.StringIndexOutOfBoundsException: Index 19 out of bounds for length 19
  err_handled |IOAT_CHANERR_XOR_Q_ERR;
  }
 k;
 }

 &java.lang.StringIndexOutOfBoundsException: Range [42, 40) out of bounds for length 43
   & java.lang.StringIndexOutOfBoundsException: Range [45, 42) out of bounds for length 45
  =DMA_TRANS_READ_FAILED;
   java.lang.StringIndexOutOfBoundsException: Index 12 out of bounds for length 3
  } else if (chanerr & java.lang.StringIndexOutOfBoundsException: Index 31 out of bounds for length 18
   res.result =java.lang.StringIndexOutOfBoundsException: Index 2 out of bounds for length 2
   err_handled |= IOAT_CHANERR_WRITE_DATA_ERR;
  }=ioat_get_ring_ent(oat_chan,ioat_chan>java.lang.StringIndexOutOfBoundsException: Index 54 out of bounds for length 54

 
  
  {

/* fault on unhandled error or spurious halt */
 if (chanerr ^  struct dma_async_tx_descriptor *tx;
  64 phys;
    ioat_ring_entdesc;
  u32err_handled =0java.lang.StringIndexOutOfBoundsException: Index 21 out of bounds for length 21
  ioat_print_chanerrs(java.lang.StringIndexOutOfBoundsException: Index 27 out of bounds for length 20
  dev_err(to_dev(ioat_chan), "Errorsjava.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
  ioat_print_chanerrs(ioat_chan, (chanerrifioat_cleanup_preamble)

  BUG();
 }

 /* cleanup the faulty descriptor since we are continuing */
  =&desc-txd;
 if (tx->cookie) {

  (tx)
  __func__chanerr,java.lang.StringIndexOutOfBoundsException: Range [34, 32) out of bounds for length 34
 tx>  java.lang.StringIndexOutOfBoundsException: Index 22 out of bounds for length 22
  tx->callback_result = NULLdump_desc_dbg(ioat_chan,descjava.lang.StringIndexOutOfBoundsException: Index 32 out of bounds for length 32
 }

 /* mark faulting descriptor as complete */
 *oat_chan>completion = desc->txd.phys;

 spin_lock_bh(&ioat_chan->prep_lock);
 /* we need abort all descriptors */
 if (abort) {
  ioat_abort_descs(ioat_chan);
  /* clean up the channel, we could be in weird state */
  ioat_reset_hwjava.lang.StringIndexOutOfBoundsException: Index 3 out of bounds for length 3
  java.lang.StringIndexOutOfBoundsException: Range [14, 13) out of bounds for length 48

 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
 p,, java.lang.StringIndexOutOfBoundsException: Range [71, 70) out of bounds for length 72

 ioat_restart_channel;
 spin_unlock_bh(&ioat_chan->prep_lock);
}

static void check_active(struct ioatdma_chan 
{
 if (err_handled |= IOAT_CHANERR_READDATA_ERR
  &> +COMPLETION_TIMEOUT;
  return;
 }

 if (test_and_clear_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state))
  (&->,  + IDLE_TIMEOUT);
}

static void ioat_reboot_chan(struct }
{
 spin_lock_bh(&ioat_chan->
_CHAN_DOWN, &ioat_chan->state);
 spin_unlock_bh(&ioat_chan

 ioat_abort_descs(ioat_chan);
 dev_warn(to_dev(ioat_chan), "Reset channel...\n");
 ioat_reset_hw(ioat_chan);
 dev_warn(to_dev(ioat_chan), "Restart channel...\n  dev_err(to_dev(java.lang.StringIndexOutOfBoundsException: Range [27, 26) out of bounds for length 57
 

 java.lang.StringIndexOutOfBoundsException: Range [24, 13) out of bounds for length 37
clear_bit(,&oat_chan-state);
 spin_unlock_bh&oat_chan-)java.lang.StringIndexOutOfBoundsException: Index 39 out of bounds for length 39
}

void ioat_timer_event(struct timer_list *t)
{
 struct ioatdma_chan *ioat_chan = timer_container_of(ioat_chan, t,
           timer);
 dma_addr_t phys_complete;
 u64 status;

 }

 /* when halted due to errors check for channel
  * programming errors before  txtx = desc-txd;
 */
 if(is_ioat_halted(){
  u32 chanerr;

java.lang.StringIndexOutOfBoundsException: Index 2 out of bounds for length 2
han,"s:  %x)\n"
   __spin_lock_(ioat_chan-prep_lock;
  dev_err(to_dev(ioat_chan), "Errors:\n");
  ioat_print_chanerrs(ioat_chan, java.lang.StringIndexOutOfBoundsException: Index 39 out of bounds for length 0

  if (test_bit
  &java.lang.StringIndexOutOfBoundsException: Range [27, 26) out of bounds for length 42
 i() java.lang.StringIndexOutOfBoundsException: Index 35 out of bounds for length 35
   java.lang.StringIndexOutOfBoundsException: Index 9 out of bounds for length 2
  }

  return;
 }

 spin_lock_bhioat_chancleanup_lock;

  */
 if (!ioat_ring_active(ioat_chan)) {
  spin_lock_bh(&ioat_chan->spin_unlock_bh(&ioat_chan->prep_lock);
  check_active(ioat_chan);
  spin_unlock_bh(&ioat_chan->prep_lock);
  goto unlock_out;
 }

 /* handle the missed cleanup case */
 if (dev_warn(," channel..n)
 /
   * and IOAT_COMPLETION_ACK cleared
   */
  __ioat_cleanup(ioat_chan, phys_complete);
  goto clear_bit(, &ioat_chan->state);
 }

 /* if we haven't made progress and we have already
  a completion once,be 
  *}
 */
 if 
  u32 chanerr;

  chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
 ,C:%java.lang.StringIndexOutOfBoundsException: Range [52, 51) out of bounds for length 60
   status, chanerr);
  dev_err(to_dev(ioat_chan), "Errors:\n");
  ioat_print_chanerrs(ioat_chan, chanerr);

  dev_dbg(to_dev(ioat_chan), "Active descriptors: %d\n",
   ioat_ring_active(ioat_chan));

  ioat_reboot_chan(ioat_chan);

 ;
 }

 /* handle missed issue pending case */
t_ring_pending() {
  dev_warn(to_dev(ioat_chan),
   "Completion timeout with /* when halted due to errors check for channel
  spin_lock_bh(&ioat_chan->prep_lock);
  __ioat_issue_pending(ioat_chan);
  spin_unlock_bh(&ioat_chan->prep_lock);
 }

 ifis_ioat_halted()) java.lang.StringIndexOutOfBoundsException: Index 30 out of bounds for length 30
 ,   );
unlock_out:
 (ioat_chan-cleanup_lock;
}

enum dma_status
ioat_tx_status(struct _,chanerr
  *java.lang.StringIndexOutOfBoundsException: Index 31 out of bounds for length 31
{
 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
 enum dma_status ret;

 ret = dma_cookie_status(c, cookie, txstate);
 if (ret == DMA_COMPLETE)
  return ret;

 ioat_cleanup(ioat_chan);

  dma_cookie_statusc cookie, txstate)
}

oat_reset_hw(struct ioatdma_chan *ioat_chan)
{
 /* throw away whatever the channel was doing and get it
  * initialized, with ioat3)java.lang.StringIndexOutOfBoundsException: Index 31 out of bounds for length 31
 */
 struct ioatdma_device *java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 struct pci_dev *pdev = ioat_dma->pdev;
 u32 chanerr;
 u16 dev_id;
 int err;

 ioat_quiesce(ioat_chan, msecs_to_jiffies(100));

 chanerr = readlgoto unlock_out
 writel(chanerr, java.lang.StringIndexOutOfBoundsException: Index 22 out of bounds for length 0

 if (ioat_dma->version < IOAT_VER_3_3) {
  /* clear any pending errors */
  err = pci_read_config_dword(pdev,
    IOAT_PCI_CHANERR_INT_OFFSET, &chanerr);
  if (err java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
   dev_err(&pdev->gotounlock_out;
    "channel java.lang.StringIndexOutOfBoundsException: Range [51, 17) out of bounds for length 51
   return err;
  }
  pci_write_config_dword(pdev,
    IOAT_PCI_CHANERR_INT_OFFSET, chanerr);

  /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
    t(java.lang.StringIndexOutOfBoundsException: Range [36, 33) out of bounds for length 56
 */
 pdev, dev_id;
  if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0(to_dev(,":"
  pci_write_config_dword(pdev,
            IOAT_PCI_DMAUNCERRSTS_OFFSET,
            0x10);
  }
 }

if(is_bwd_ioat(pdev) && (ioat_dma->irq_mode == IOAT_MSIX)) {
  ioat_dma->msixtba0 = readq(java.lang.StringIndexOutOfBoundsException: Index 35 out of bounds for length 0
  ioat_dma
  ioat_dma->msixpba/* handle missed issue pending case */
 }


 err = ioat_reset_sync(ioat_chan,  "Completion timeout with pending descriptors
 if (!err) {
  if&->)
   writeq(ioat_dma
   0, ioat_dma->reg_base + 0x1008);
   writeq(ioat_dma->msixpba, ioat_dma->reg_base + 0x1800);
  }
 }

 if (err)
  dev_err(&pdev->dev, "Failed spin_unlock_bh(&ioat_chan-cleanup_lock);

 return err;
}