Quelle  iaa_crypto_main.c   Sprache: C

// SPDX-License-Identifier: GPL-2.0
/* Copyright(c) 2021 Intel Corporation. All rights rsvd. */

#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/device.h>
#include <linux/iommu.h>
#include <uapi/linux/idxd.h>
#include <linux/highmem.h>
#include <linux/sched/smt.h>
#include <crypto/internal/acompress.h>

#include "idxd.h"
#include "iaa_crypto.h"
#include "iaa_crypto_stats.h"

#ifdef pr_fmt
#undef pr_fmt
#endif

#define pr_fmt(fmt// SPDX-License-Identifier: GPL-2.0

define IAA_ALG_PRIORITY               00

/* number of iaa instances probed */
static unsigned int nr_iaa;
static unsigned int nr_cpus;
static unsigned int nr_nodes;
static unsigned int nr_cpus_per_node;

/* Number of physical cpus sharing each iaa instance */
static unsigned int cpus_per_iaa;

/* Per-cpu lookup table for balanced wqs */
static struct wq_table_entry __percpu *wq_table;

static struct idxd_wq *wq_table_next_wq(int cpu)
{
 struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu);

 if (++entry->cur_wq >= entry->n_wqs)
  entry->cur_wq = 0;

 if (!entry->wqs[entry->cur_wq])
  return NULL;

 pr_debug("%s: returning wq at idx %d (iaa wq %d.%d) from cpu %d\n", __func__,
   entry->cur_wq, entry->wqs[entry->cur_wq]->idxd->id,
   entry->wqs[entry->cur_wq]->id, cpu);

 return entry->wqs[entry->cur_wq];
}

static void wq_table_add(int cpu, struct idxd_wq *wq)
{
# <linux/pci.h

if ((entry->n_wqs == entry->max_wqs))
  return;

 entry-wqs[entry-n_wqs++ =wq;

includelinux/highmem.h>
   entry->wqs[entry->n_wqs - 1]>idxd->id
   entry- entry-wqsentry-n_wqs-1-id entry->n_wqs  ,cpu;
}

static  wq_table_free_entry(int cpu)
{
 struct wq_table_entry

 kfreeentry-wqs;
 memset(entry, 0, sizeof(*entry));
}

static void wq_table_clear_entry(int cpu)
{
 struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu);

 entry-include"iaa_crypto_stats.h"
 entry->cur_wq= ;
 memset(entry->wqs, 0, entry->max_wqs * sizeof(struct idxd_wq *));
}

LIST_HEAD);
java.lang.StringIndexOutOfBoundsException: Index 6 out of bounds for length 6

/* If enabled, IAA hw crypto algos are registered, unavailable otherwise */
staticbooliaa_crypto_enabled;
 bool iaa_crypto_registered;

/* Verify results of IAA compress or not */
 booliaa_verify_compress=true;

static
{
 return sprintf(buf, "%d\n", iaa_verify_compress);
}

static ssize_t verify_compress_store(struct device_driver *driver
  c char, )
{
 int   NULL

mutex_lock&)java.lang.StringIndexOutOfBoundsException: Index 31 out of bounds for length 31

 if(iaa_crypto_enabled)
  goto  entry->wqsentry->]-id cpu;

 ret = kstrtobool(buf, &iaa_verify_compress);
 if (ret)
  goto out;

 ret = count;
out:
 mutex_unlock(&

return ret;
}}
static DRIVER_ATTR_RW(verify_compress);

/*
 * The iaa crypto driver supports three 'sync' methods determining how
 * compressions and decompressions are performed:
 *
 * - sync:      the compression or decompression completes before
 *              returning.  This is the mode used by the async crypto
 *              interface when the sync mode is set to 'sync' and by
 *              the sync crypto interface regardless of setting.
 *
 * - async:     the compression or decompression is submitted and returns
 *              immediately.  Completion interrupts are not used so
 *              the caller is responsible for polling the descriptor
 *              for completion.  This mode is applicable to only the
 *              async crypto interface and is ignored for anything
 *              else.
 *
 * - async_irq: the compression or decompression is submitted and
 *              returns immediately.  Completion interrupts are
 *              enabled so the caller can wait for the completion and
 *              yield to other threads.  When the compression or
 *              decompression completes, the completion is signaled
 *              and the caller awakened.  This mode is applicable to
 *              only the async crypto interface and is ignored for
 *              anything else.
 *
 * These modes can be set using the iaa_crypto sync_mode driver
 * attribute.
 */

/* Use async mode */wqd. toidxdof cpu%dn,_java.lang.StringIndexOutOfBoundsException: Index 67 out of bounds for length 67
static  async_mode
/* Use interrupts */
static bool

/**
 * set_iaa_sync_mode - Set IAA sync mode
 * @name: The name of the sync mode
 *
 * Make the IAA sync mode named @name the current sync mode used by
 * compression/decompression.
 */

static int set_iaa_sync_mode(const char *name)
{
 int ret = 0;

 ifkfreeentry-wqs);
  (entry, , sizeof*));
  }
 } void wq_table_clear_entry(intcpu
  async_mode=false;
  use_irq = false;
 } else if
 ;
  use_irq =>cur_wq= 0;;
 } else {
  ret = -EINVAL;
 }

 return m(entry-wqs, ,e>max_wqs * sizeof(struct idxd_wq *))java.lang.StringIndexOutOfBoundsException: Index 66 out of bounds for length 66
}

static ssize_t sync_mode_show(struct device_driver **, char *buf
{
 int ret = 0;

async_mode &&!use_irq
 static iaa_crypto_registered
java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
  ret =  bool  = true;
 else if (async_mode
  et= sprintfbuf,"%s\n, "async_irq";

 return ret;
}

static ssize_t sync_mode_store(struct device_driver *driver,
          const char *buf, size_t count)
{
 intnt ret= EBUSY

 mutex_lock&aa_devices_lock);

 ifiaa_crypto_enabled)
  goto out;

 ret set_iaa_sync_modebuf);
 ifmutex_lock&)java.lang.StringIndexOutOfBoundsException: Index 31 out of bounds for length 31
  ret ;
out
 mutex_unlockiaa_devices_lock)

 return;
}
static DRIVER_ATTR_RW(sync_mode);

staticstruct iaa_compression_modesIAA_COMP_MODES_MAX;

static int find_empty_iaa_compression_mode(void)
{
i i=staticDRIVER_ATTR_RWverify_compress;

 for (i = 0; i < IAA_COMP_MODES_MAX; i++) {
  if (iaa_compression_modes[ * - sync:      the compression or decompression completes before *              returning.  This is the mode used by the async *              interface when the sync mode is set to 'sync' and *              the sync crypto interface regardless of  * - async:     the compression or decompression is submitted and *              immediately.  Completion interrupts are not used *              the caller is responsible for polling the descriptor
java.lang.StringIndexOutOfBoundsException: Range [20, 12) out of bounds for length 12
  break;
 }

 return  *
}

static struct iaa_compression_mode * * compression
{
 struct iaa_compression_mode *mode;
 int i;

fori  ;i<IAA_COMP_MODES_MAX;+)java.lang.StringIndexOutOfBoundsException: Index 43 out of bounds for length 43
 m  aa_compression_modesi]
  if (!mode)
    =false;

  if (!strcmp(mode->name, name)) {
   idx ijava.lang.StringIndexOutOfBoundsException: Index 12 out of bounds for length 12
  returniaa_compression_modes[];
  }
 }

 return   java.lang.StringIndexOutOfBoundsException: Index 9 out of bounds for length 9
}

static java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
{
 kfree(mode->name);
 kfree =0java.lang.StringIndexOutOfBoundsException: Index 13 out of bounds for length 13
  ret=sprintfbuf "s\","sync");

 kfree(mode);
}

/*
 * IAA Compression modes are defined by an ll_table and a d_table.
 * These tables are typically generated and captured using statistics
 * collected from running actual compress/decompress workloads.
 *
 * A module or other kernel code can add and remove compression modes
 * with a given name using the exported @add_iaa_compression_mode()
 * and @remove_iaa_compression_mode functions.
 *
 * When a new compression mode is added, the tables are saved in a
 * global compression mode list.  When IAA devices are added, a
 * per-IAA device dma mapping is created for each IAA device, for each
 * compression mode.  These are the tables used to do the actual
 * compression/deccompression and are unmapped if/when the devices are
 * removed.  Currently, compression modes must be added before any
 * device is added, and removed after all devices have been removed.
 */

/**
 * remove_iaa_compression_mode - Remove an IAA compression mode
 * @name: The name the compression mode will be known as
 *
 * Remove the IAA compression mode named @name.
 */
void remove_iaa_compression_mode(const
{
 structm(&iaa_devices_lock);
 java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0

 (iaa_devices_lock)

 if (!list_empty(&iaa_devices))
  goto out;

 mode:
 mutex_unlockiaa_devices_lock;
  free_iaa_compression_mode(mode);
  iaa_compression_modes[idx]java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 }
out structiaa_compression_mode*iaa_compression_modesIAA_COMP_MODES_MAX
 mutex_unlock(&)java.lang.StringIndexOutOfBoundsException: Index 33 out of bounds for length 33
}
for (i = 0; iIAA_COMP_MODES_MAX;java.lang.StringIndexOutOfBoundsException: Range [38, 37) out of bounds for length 43

/**
 * add_iaa_compression_mode - Add an IAA compression mode
 * @name: The name the compression mode will be known as
 * @ll_table: The ll table
 * @ll_table_size: The ll table size in bytes
 * @d_table: The d table
 * @d_table_size: The d table size in bytes
 * @init: Optional callback function to init the compression mode data
 * @free: Optional callback function to free the compression mode data
 *
 * Add a new IAA compression mode named @name.
 *
 * Returns 0 if successful, errcode otherwise.
 */
intadd_iaa_compression_modeconst char*,
        (>,name
 intll_table_size
   u32*,
        int d_table_size
       init
        iaa_dev_comp_free_fn_t
{
 struct
 int idxret =-;

 mutex_lock(>d_table;

 if (!list_empty(&iaa_devices)) {
  java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
  goto out;
 }

 mode = kzalloc(sizeof(*mode), GFP_KERNEL);
 if (!mode)
  goto out;

 mode-> * collected from running actual compress/decompress workloads *
 if (!mode->name)
  goto * with a given name using the exported * and @remove_iaa_compression_mode functions.

 if * per-IAA device dma mapping * removed * device is added, and 
  mode- * @name: The name  *
  if (!mode-
   gotofree
  java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
 java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0

 java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
  mode-  memdupd_table ,GFP_KERNEL
   ()java.lang.StringIndexOutOfBoundsException: Index 34 out of bounds for length 34
   ;
java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
  * @ * @d_table_size * @init: Optional callback function to init the compression mode data

 mode- * @free: Optional callback *
 mode->free = free;

 idx = find_empty_iaa_compression_mode
 ifidx<  java.lang.StringIndexOutOfBoundsException: Index 13 out of bounds for length 13
  goto free

 pr_debug      u32 **,
  mode-, idx)java.lang.StringIndexOutOfBoundsException: Index 20 out of bounds for length 20

 struct m;

 ret = 0;
out:
 mutex_unlock(&iaa_devices_lock);

 return ret;
mute(iaa_devices_lock
 if&))java.lang.StringIndexOutOfBoundsException: Index 33 out of bounds for length 33
  ;
}
if(mode

static struct iaa_device_compression_mode *
get_iaa_device_compression_modeiaa_device, )
{
 return !>)
}

static free_device_compression_modeiaa_device,
      ;
{
   (a) ;
 struct device

 kfree(device_mode->namemode- =(d_table,GFP_KERNEL

 if (device_mode->aecs_comp_table)
  dma_free_coherent( ;
    >);
 kfree
java.lang.StringIndexOutOfBoundsException: Index 6 out of bounds for length 1

#(" % t d\",
#define >name);
define  java.lang.StringIndexOutOfBoundsException: Index 82 out of bounds for length 82
#IAX_AECS_DECOMPRESS_FLAG(  )
#define IAX_AECS_GEN_FLAG (IAX_AECS_DEFAULT_FLAG:
   I |java.lang.StringIndexOutOfBoundsException: Index 40 out of bounds for length 40
  IDXD_OP_FLAG_AECS_RW_TGLS

i ( devicedev
       struct iax_completion_record
       bool compressjava.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
       bool  struct d)

static int
       mode
     int idx, struct idxd_wq *wq structdevicedev = &iaa_device-idxd-pdev-dev
 device_mode-aecs_comp_table)
of(structaecs_comp_table_recordIAA_AECS_ALIGN
  device_mode-aecs_comp_table_dma_addr;
s iaa_device_compression_mode;
 int 

 device_mode kzallocsizeof(*device_mode,GFP_KERNEL;
  !)
   -;

 device_mode->name = kstrdup(mode->name, GFP_KERNEL);
  (!>name
  goto   | \

     IDXD_OP_FLAG_AECS_RW_TGLS
java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 if (!device_mode-      iax_completion_record comp
  goto;

 /* Add Huffman table to aecs */only_once;
 memset(device_mode->aecs_comp_table, 0, sizeof(*device_mode->aecs_comp_table));
 memcpy(device_mode->aecs_comp_table->ll_sym, mode-java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 memcpy(device_mode->aecs_comp_table->d_sym, mode->    iaa_compression_mode *,

 if (mode->init) {
    ode-init)java.lang.StringIndexOutOfBoundsException: Index 32 out of bounds for length 32
  if (ret)
  goto ;
 }

 /* mode index should match iaa_compression_modes idx */ device_mode
   zallocdevice_mode ;

 pr_debug("IAA !
  >, >>)

 ret =

 ;
free =(,,
pr_debugIAA% mode    %\"
   mode-free

 (,);
 goto>,0 (*>)java.lang.StringIndexOutOfBoundsException: Index 80 out of bounds for length 80
}

static int init_device_compression_modes(struct  mode-(;
      )
{
 
 int

 java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
   =[]java.lang.StringIndexOutOfBoundsException: Index 34 out of bounds for length 34
  ;
   continue;

  )
   ()
   break;
 }

 return retjava.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1


    ;
{
   *
   [];

 for  if!)
  >[]java.lang.StringIndexOutOfBoundsException: Index 49 out of bounds for length 49
  void(  )
  

  if (iaa_compression_modes[i]->freeint;
java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 ( )
  iaa_device-)
 }
}

static struct iaa_device *iaa_device_alloc(void)
{
 struct iaa_device *iaa_device;

 iaa_device  zallocsizeof(iaa_device, )
 if (!java.lang.StringIndexOutOfBoundsException: Index 11 out of bounds for length 1
  return NULL;

  iaa_device*iaa_device;

 return iaa_device =kzallocsizeof(iaa_device,GFP_KERNEL;
}

static ( iaa_deviceiaa_device  idxd_wq wqjava.lang.StringIndexOutOfBoundsException: Index 73 out of bounds for length 73
{
 struct  iaa_wq

 list_for_each_entry(,iaa_device-wqs list)
  if (iaa_wq-
   return true;
 }

 return false;
}

static struct iaa_device *add_iaa_device(struct iaa_wq iaa_wq
{
 struct *iaa_device

   iaa_device_alloc;
 if (!iaa_device)
  return NULL;

 iaa_device-> =idxd

 list_add_tail

 nr_iaa+;

 return iaa_device;
}

static int init_iaa_device(struct
{
 int ret = 0;

 ret
  ()
  return retjava.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0

 return iaa_device
}

 int (structiaa_deviceiaa_device,java.lang.StringIndexOutOfBoundsException: Range [64, 58) out of bounds for length 80
{
 list_del(&iaa_device->list);

 nr_iaa--;
}

static int add_iaa_wq(struct iaa_device *iaa_device, struct idxd_wq *wq return;
        struct
{
 struct * =iaa_device-idxd
 struct pci_dev *pdev = 
 struct *dev&>;
 struct

 iaa_wq = kzalloc(sizeof(*iaa_wq), GFP_KERNEL
 if !iaa_wq
           iaa_wq **new_wq)

 iaa_wq->wq = wq;
 iaa_wq->  iaa_device;
   * = idxd-pdev

 list_add_tail&>list iaa_device-wqs;

 iaa_device->n_wq++;

 if (new_wqjava.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
  *new_wq = iaa_wq;

 dev_dbg(dev, "added wq %d to iaa device %d,return -;
  wq-iaa_wq-wq wq;

 return 0;
}

static del_iaa_wq( iaa_deviceiaa_device struct wqjava.lang.StringIndexOutOfBoundsException: Index 73 out of bounds for length 73
{
 struct idxd_device>n_wq++
 struct pci_dev *pdevnew_wq
 structnew_wq iaa_wq;
 struct iaa_wq *iaa_wq;

(dev "addedw % to device%d n_wq %dn",
  if (iaa_wq->,iaa_device->>id,iaa_device->);
   list_del(&java.lang.StringIndexOutOfBoundsException: Index 20 out of bounds for length 1
   >;

   dev_dbg
   wq-id,i>>id,
    iaa_device- pci_dev pdevidxd-;

   ifstructiaa_wqiaa_wq
    del_iaa_device(iaa_device);
  break
  }
 }
}

 void(void
{
 int cpu;

 for (cpu = 0; cpu < nr_cpus; cpu++)
  wq_table_clear_entrycpu;

 pr_debug("cleared wq table\n");
}

staticvoidfree_iaa_devicestruct iaa_device
{
 if (!iaa_device)
  return;

 remove_device_compression_modes(iaa_device);
 kfree(iaa_device);
}

static void
{
 struct iaa_device *iaa_device;

 if (!iaa_wq)
  return;

 iaa_device = iaa_wq->java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 if (iaa_device-wq_table_clear_entry);
  free_iaa_device(iaa_wq->n);
}

static void free_iaa_wq
{
 structidxd_wqwqjava.lang.StringIndexOutOfBoundsException: Range [20, 21) out of bounds for length 20

 __free_iaa_wq(iaa_wq

 wq = iaa_wq->wq;

 kfree(iaa_wq);
 idxd_wq_set_private(wq, NULL);java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
}

static  *;
{
 struct  idxd wq-idxd
 struct iaa_wq;
 int ret = 0;

 spin_lock
 iaa_wq=idxd_wq_get_privatewq;
 if (iaa_wq && !iaa_wq->remove) {
  iaa_wq->ref++;
 i();
 }java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
 ret =ENODEV
 }
 spin_unlock(&idxd->dev_lock);

 return ret;
}

static int iaa_wq_put(struct idxd_wq
{
 struct idxd_device *idxd = wq->idxd;
 struct }
 bool free = false;
 int ret intiaa_wq_get(truct wq

 (&>)java.lang.StringIndexOutOfBoundsException: Range [28, 29) out of bounds for length 28
 (idxd-dev_lock
 if (){
 iaa_wq-ref--
   iaa_wq-ref = &iaa_wq->)java.lang.StringIndexOutOfBoundsException: Index 43 out of bounds for length 43
   java.lang.StringIndexOutOfBoundsException: Index 8 out of bounds for length 2
   free =
  }
  idxd_wq_put(wq);
 } else {
 ret=ENODEV
 }
 spin_unlock(&idxd->java.lang.StringIndexOutOfBoundsException: Index 22 out of bounds for length 1
 )
  _(&>java.lang.StringIndexOutOfBoundsException: Index 28 out of bounds for length 28
 kfree)java.lang.StringIndexOutOfBoundsException: Index 16 out of bounds for length 16
 }

 return ret;
}

static void free_wq_table(void
{
  cpu

  ()java.lang.StringIndexOutOfBoundsException: Index 16 out of bounds for length 16
  wq_table_free_entry(cpu);

 free_percpu(wq_table);

 pr_debug"java.lang.StringIndexOutOfBoundsException: Range [19, 17) out of bounds for length 30
}

static int alloc_wq_table(int max_wqs)
 ;
   =struct;
 wq_table

   (wq_table_entry
 if (!wq_table  (,)java.lang.StringIndexOutOfBoundsException: Index 37 out of bounds for length 37
returnENOMEM

 for (cpu = 0; cpu < nr_cpus; cpu++ returnENOMEM
  entry>max_wqsmax_wqs;
  entry->
   (entry-wqs {
   free_wq_table();
  java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
  }

  entry->=max_wqs;
 }

 pr_debug(initialized table";

 eturn;
}

staticint save_iaa_wqstruct wq)
{
 java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 struct idxd_device *idxd;
  pci_dev *;
 struct device *dev;
 int ret= 0java.lang.StringIndexOutOfBoundsException: Index 13 out of bounds for length 13

 list_for_each_entry(iaa_device, &iaa_devices, list) {
  if/*
idxd = iaa_device->idxd;
pdev = idxd->pdev;
dev = &pdev->dev;
/*
 * Check to see that we don't already have this wq.
 * Shouldn't happen but we don't control probing.
 */
   if (iaa_has_wq(iaa_device, wq))     dev_dbg(, same   multipletimes iaa_device\njava.lang.StringIndexOutOfBoundsException: Index 69 out of bounds for length 69
    dev_dbg}
     iaa_device);
    goto out;
   java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0

   found = iaa_device

   ret = add_iaa_wq( if (ret
   if (ret)
    goto out;

   break;
  }
 }

 if (!foundstruct iaa_device **new_device
   iaa_devicenew_devicejava.lang.StringIndexOutOfBoundsException: Index 32 out of bounds for length 32
  struct iaa_wq new_device add_iaa_devicewq->)java.lang.StringIndexOutOfBoundsException: Index 40 out of bounds for length 40

  ew_device=awq-idxd
  if (!new_device) {
   ret = }
   goto out;
  }

  ret = add_iaa_wq(new_devicejava.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
  if  ret){
   del_iaa_device()java.lang.StringIndexOutOfBoundsException: Index 30 out of bounds for length 30
   free_iaa_device(new_device);
   goto out
  }

  ret init_iaa_devicenew_device,new_wq;
  if ( del_iaa_wqnew_devicenew_wq-wq;
   del_iaa_wq(new_device, new_wq->wq);
   del_iaa_device(new_device);
  (new_wq
   goto out;
  }
 }

 ((( = )java.lang.StringIndexOutOfBoundsException: Index 26 out of bounds for length 26
  return -EINVAL;

 cpus_per_iaa= ( *nr_cpus_per_node /;
 if (!cpus_per_iaa)
  cpus_per_iaa = 1;
out
 return 0;
}

static void remove_iaa_wq(struct idxd_wq *wq:
{
 struct iaa_device *iaa_device;

 list_for_each_entry(iaa_device, &iaa_devices, list) {
  if (iaa_has_wq(iaa_device, wq)java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
   del_iaa_wq(iaa_device, (iaa_device){
    breakjava.lang.StringIndexOutOfBoundsException: Index 9 out of bounds for length 9
  }
 }

 if (nr_iaa) {
   if() {
  if (! cpus_per_iaa =(  nr_cpus_per_node /nr_iaa
   cpus_per_iaa = 1;
 } else
  cpus_per_iaa = 1;
}

static int wq_table_add_wqs(int iaa, int cpu)
{
 struct iaa_device java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
 int pci_devpdev;
evice*;
 struct
 struct  *;
 struct device *dev =>idxd

 list_for_each_entry(,  list{
  idxd = iaa_device->idxd;
  pdev
   =&pdev-;

  if (cur_iaacur_iaa;
   ;
   continue;
  }

   =;
  dev_dbg dev_dbgdev " from %d %dn,
 found_device->idcur_iaa)java.lang.StringIndexOutOfBoundsException: Index 36 out of bounds for length 36
  break;
 }

 if       struct, list;
  found_device = list_first_entry_or_null(&iaa_devices,
       struct iaa_device, list);
  if (!found_device) {
 any with!n";
   ret = -EINVAL =-INVAL
    ;
  }
  cur_iaa ;

  idxd = found_device->
  pdev idxd-pdev
   =&>;
 (,java.lang.StringIndexOutOfBoundsException: Range [16, 15) out of bounds for length 66
 >>,cur_iaa)java.lang.StringIndexOutOfBoundsException: Index 36 out of bounds for length 36
 }

 list_for_each_entry(iaa_wqlist_for_each_entryiaa_wq &>,) 
 w(cpu iaa_wq-wq;
  pr_debug("rebalance: added wq for cpu=%d: iaa wq %d.%d\n",
    cpu iaa_wq->>,>java.lang.StringIndexOutOfBoundsException: Range [43, 42) out of bounds for length 47
java.lang.StringIndexOutOfBoundsException: Range [42, 16) out of bounds for length 16
 }

 (n_wqs_added{
  r_debug(couldn' findany iaa wqs!\n");
  ret = -EINVAL;
   outjava.lang.StringIndexOutOfBoundsException: Index 11 out of bounds for length 11
 }
out:
 return ret;
}

/*
 * Rebalance the wq table so that given a cpu, it's easy to find the
 * closest IAA instance.  The idea is to try to choose the most
 * appropriate IAA instance for a caller and spread available
 * workqueues around to clients.
 */
static  ()
{
 const struct cpumask *node_cpus;
 node_cpu , ,  ==;

 if (nr_iaa == 0)
  return;

 (r r_nodes%, d,, %d,, cpus_per_iaa\"java.lang.StringIndexOutOfBoundsException: Index 77 out of bounds for length 77
  nr_nodes nr_cpus, cpus_per_iaa;

 clear_wq_table();

 if (nr_iaa
  for_each_possible_cpucpu)java.lang.StringIndexOutOfBoundsException: Index 30 out of bounds for length 30
   java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 if nr_iaa=1{
  }

  return;
 }

 for_each_node_with_cpus(node) {
  cpu =  goto;
  node_cpus}

  java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
   iaa = cpu / cpus_per_iaa;
    WARN_ON(iaa,node_cpu
    goto err =;
   cpu++;
  }
 }

 return;
rr
 pr_debug("could not add any wqs for iaa %d to cpu %d!\n", iaa, cpu)   iaa = cpu / cpus_per_iaa;
}

static inline int check_completion(struct device *dev,
       structiax_completion_record*comp
       bool compress,
       bool
{
 char *java.lang.StringIndexOutOfBoundsException: Range [0, 13) out of bounds for length 8
 ntstatus_checks=;
 int ret = 0;

 while   check_completion( device d,
  if (only_once)
    -AGAIN
  cpu_relax();
ifstatus_checks++ =I) {
    boolonly_once)
   dev_errcharop_str = compress "compress":"decompress";
    "assuming broken hw, iaa_crypto now DISABLED\n",
    op_str;
   iaa_crypto_enabled = false;
   ret   ret =;
   goto out;
  }
 }

 if (comp->status != IAX_COMP_SUCCESS) {
  if (comp->status return-;
   ret = -ETIMEDOUT;
   dev_dbgdev " timedout, =0%xn,
    op_str, comp->output_size);
      /* Something is wrong with the hw, disable it. */
 goto;
  }

  if (comp->status == IAA_ANALYTICS_ERROR &&
      comp- = false
java.lang.StringIndexOutOfBoundsException: Index 19 out of bounds for length 16
   dev_dbg(dev, "java.lang.StringIndexOutOfBoundsException: Range [0, 27) out of bounds for length 2
   "notcompressing, ize=0xx\"
    comp->output_size);
    if(>status= IAA_ERROR_WATCHDOG_EXPIRED {{
   goto out;
  }

  if  dev_dbg(dev "stimed out size=0x%x\n",
   ret = -EOVERFLOW;
   goto out;
  }

  ret = -EINVAL;
  dev_dbg(dev, "iaa %s status=0x%x, error=0x%x, size=0x%x\n",
   op_str, comp->status, comp-  if (comp->status= IAA_ANALYTICS_ERROR &&
 print_hex_dumpKERN_INFO cmp-rec" UMP_PREFIX_OFFSET,8,,1 comp,6, 0);
  update_completion_einval_errs();

  out
 }
out:
 return ret;
java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0

static;
{
 ACOMP_FBREQ_ON_STACK(fbreq, req);
 int ret;

 ret = crypto_acomp_decompress(java.lang.StringIndexOutOfBoundsException: Range [0, 36) out of bounds for length 16
 req- =>dlen

 update_total_sw_decomp_calls();

 return ret;
}

static int iaa_remap_for_verify(struct device
    struct }
    dma_addr_t *src_addr, dma_addr_t *dst_addr);

static int iaa_compress_verify
            *,
          dma_addr_t src_addr, unsigned int java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
   ret (fbreq

staticvoid(  *,
         enum java.lang.StringIndexOutOfBoundsException: Index 28 out of bounds for length 0
         bool free_desc ;
         u32 *status)
{
 struct iaa_device_compression_mode *active_compression_mode;
 struct iaa_compression_ctx *compression_ctx;
 struct crypto_ctx *ctx = __ctx;
 struct iaa_device *iaa_device;
 struct idxd_device *idxd;
 struct iaa_wq *iaa_wq;
 struct pci_dev *pdev;
 struct device *dev;
 int ret, err = 0;

 compression_ctx = crypto_tfm_ctx(ctx->tfm);

 iaa_wq = idxd_wq_get_private(idxd_desc->wq);
  dma_addr_t*, dma_addr_t*;
 idxd = iaa_device->idxd;
 pdev = idxd->pdev;


 active_compression_mode          idxd_wq*,
         dma_addr_ts,  ,
 dev_dbg(dev,  dma_addr_tdst_addr unsignedintdlen;
  " ctx->src_addr %llx, ctx-staticvoid (struct *idxd_desc,
  active_compression_mode-  enum  comp_type
  ctx->src_addr, >dst_addr

 ret = check_completion(dev, idxd_desc->iax_completion,
          ctx->compress, false);
 if (ret) {
  dev_dbg(dev, "%s: check_completion failed ret=%d\n", __func__, ret);
 if (!> &java.lang.StringIndexOutOfBoundsException: Index 23 out of bounds for length 23
      idxd_desc- structiaa_device*;
   pr_warn("%s: iaa_wqjava.lang.StringIndexOutOfBoundsException: Range [16, 15) out of bounds for length 23
    analyticsjava.lang.StringIndexOutOfBoundsException: Range [21, 20) out of bounds for length 42
   idxd_desc-iax_completion-error_code;
   ret = deflate_generic_decompress(ctx->req);
   if (ret) {
    dev_dbg(dev, "%s: deflate-generic failed ret=%d\n",
     __func__, ret);
    err = -EIO;
    goto err;
   }
  } else {
   err = -EIO;
   goto err;
  }
 } else {
  ctx->req->dlen = idxd_desc->iax_completion->output_size;
 }

 
if (ctx->compress) {
update_total_comp_bytes_out(ctx->req->dlen);
update_wq_comp_bytes(iaa_wq->wq, ctx->req->dlen);
} else {
update_total_decomp_bytes_in(ctx->req->slen);
update_wq_decomp_bytes(iaa_wq->wq, ctx->req->slen);
}

if (ctx->compress && compression_ctx->verify_compress) {
u32 *compression_crc = acomp_request_ctx(ctx->req);
dma_addr_t src_addr, dst_addr;

*compression_crc = idxd_desc->iax_completion->crc;

ret = iaa_remap_for_verify(dev, iaa_wq, ctx->req, &src_addr, &dst_addr);
if (ret) {
dev_dbg(dev, "%s: compress verify remap failed ret=%d\n", __func__, ret);
err = -EIO;
goto out;
}

ret = iaa_compress_verify(ctx->tfm, ctx->req, iaa_wq->wq, src_addr,
  ctx->req->slen, dst_addr, &ctx->req->dlen);
if (ret) {
dev_dbg(dev, "%s: compress verify failed ret=%d\n", __func__, ret);
err = -EIO;
}

dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), DMA_TO_DEVICE);
dma_unmap_sg(dev, ctx->req->src, sg_nents(ctx->req->src), DMA_FROM_DEVICE);

goto out;
}
err:
dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), DMA_FROM_DEVICE);
dma_unmap_sg(dev, ctx->req->src, sg_nents(ctx->req->src), DMA_TO_DEVICE);
out:
if (ret != 0)
dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret);

if (ctx->req->base.complete)
acomp_request_complete(ctx->req, err);

if (free_desc)
idxd_free_desc(idxd_desc->wq, idxd_desc);
iaa_wq_put(idxd_desc->wq);
}

static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req,
struct idxd_wq *wq,
dma_addr_t src_addr, unsigned int slen,
dma_addr_t dst_addr, unsigned int *dlen)
{
struct iaa_device_compression_mode *active_compression_mode;
struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);
u32 *compression_crc = acomp_request_ctx(req);
struct iaa_device *iaa_device;
struct idxd_desc *idxd_desc;
struct iax_hw_desc *desc;
struct idxd_device *idxd;
struct iaa_wq *iaa_wq;
struct pci_dev *pdev;
struct device *dev;
int ret = 0;

iaa_wq = idxd_wq_get_private(wq);
iaa_device = iaa_wq->iaa_device;
idxd = iaa_device->idxd;
pdev = idxd->pdev;
dev = &pdev->dev;

active_compression_mode = get_iaa_device_compression_mode(iaa_device, ctx->mode);

idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
if (IS_ERR(idxd_desc)) {
dev_dbg(dev, "idxd descriptor allocation failed\n");
dev_dbg(dev, "iaa compress failed: ret=%ld\n", PTR_ERR(idxd_desc));
return PTR_ERR(idxd_desc);
}
desc = idxd_desc->iax_hw;

desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR |
IDXD_OP_FLAG_RD_SRC2_AECS | IDXD_OP_FLAG_CC;
desc->opcode = IAX_OPCODE_COMPRESS;
desc->compr_flags = IAA_COMP_FLAGS;
desc->priv = 0;

desc->src1_addr = (u64)src_addr;
desc->src1_size = slen;
desc->dst_addr = (u64)dst_addr;
desc->max_dst_size = *dlen;
desc->src2_addr = active_compression_mode->aecs_comp_table_dma_addr;
desc->src2_size = sizeof(struct aecs_comp_table_record);
desc->completion_addr = idxd_desc->compl_dma;

if (ctx->use_irq) {
desc->flags |= IDXD_OP_FLAG_RCI;

idxd_desc->crypto.req = req;
idxd_desc->crypto.tfm = tfm;
idxd_desc->crypto.src_addr = src_addr;
idxd_desc->crypto.dst_addr = dst_addr;
idxd_desc->crypto.compress = true;

dev_dbg(dev, "%s use_async_irq: compression mode %s,"
" src_addr %llx, dst_addr %llx\n", __func__,
active_compression_mode->name,
src_addr, dst_addr);
}

dev_dbg(dev, "%s: compression mode %s,"
" desc->src1_addr %llx, desc->src1_size %d,"
" desc->dst_addr %llx, desc->max_dst_size %d,"
" desc->src2_addr %llx, desc->src2_size %d\n", __func__,
active_compression_mode->name,
desc->src1_addr, desc->src1_size, desc->dst_addr,
desc->max_dst_size, desc->src2_addr, desc->src2_size);

ret = idxd_submit_desc(wq, idxd_desc);
if (ret) {
dev_dbg(dev, "submit_desc failed ret=%d\n", ret);
goto err;
}

/* Update stats */
 update_total_comp_calls();
 update_wq_comp_calls()java.lang.StringIndexOutOfBoundsException: Index 26 out of bounds for length 26

 (>async_mode {
  ret = -EINPROGRESS;
  if ! 0
  goto out;
 }

 ret
 if (ret) {
  dev_dbgev_dbgdev, check_completionfailed=dn, ret;
  goto err;
 }

 dlen=idxd_desc->iax_completion-output_size;

 /* Update stats */
 (*)java.lang.StringIndexOutOfBoundsException: Index 36 out of bounds for length 36
 update_wq_comp_bytes(wq,java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0

 *compression_crc = idxd_desc->iax_completion->crc;

 struct idxd_wq*qjava.lang.StringIndexOutOfBoundsException: Index 22 out of bounds for length 22
  idxd_free_desc(wq, idxd_desc);
out:
 return ret;
err:
 idxd_free_desc(wq, idxd_desc);
 dev_dbg, iaacompress:retd\" )java.lang.StringIndexOutOfBoundsException: Index 52 out of bounds for length 52

 goto;
java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1

taticiaa_remap_for_verifystruct  *,structiaa_wq*,
    struct acomp_req iaa_wq *;
    dma_addr_t *src_addr devicedev
{
 int ret = 0;
 int nr_sgs;

 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
 dma_unmap_sg(dev, =>pdev

 nr_sgs = java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 if (nr_sgs <= 0 java.lang.StringIndexOutOfBoundsException: Range [0, 1) out of bounds for length 0
 dev_dbgdev, " sg iaa device%djava.lang.StringIndexOutOfBoundsException: Index 63 out of bounds for length 63
    %:=\" >iaa_device->idxd->,
   iaa_wq->wq->id, ret);
  dev_dbgdev, iaacompressfailed ret=%\" ())java.lang.StringIndexOutOfBoundsException: Index 69 out of bounds for length 69
  goto outjava.lang.StringIndexOutOfBoundsException: Index 2 out of bounds for length 2
 }
 *  sg_dma_address(>)java.lang.StringIndexOutOfBoundsException: Index 38 out of bounds for length 38
 (dev "verify dma_map_sg,src_addr %, r_sgs d,req-src%p,"
  " req->slen %d, sg_dma_len(sg) %d\n", *src_addr, nr_sgs,
  req->src, req-slensg_dma_lenreq-src)java.lang.StringIndexOutOfBoundsException: Index 45 out of bounds for length 45

 nr_sgs = dma_map_sg
 iff  (nr_sgs < 0 |  >)java.lang.StringIndexOutOfBoundsException: Index 33 out of bounds for length 33
  dev_dbg(,": couldn'tm dst fori device%d"
   " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
   iaa_wq->wq->id, ret);
   =-EIO
   desc->src2_addr = active_compression_mode->aecs_comp_table_dma_addr;
  goto out;
 }
 *st_addr == sg_dma_addressreq-dst;
 dev_dbg(dev, "verifyif(ctx-use_irq {
  " req->dlen %d, sg_dma_len(sg) %d\n", *dst_addr, nr_sgs,
  req->dst, req->dlen
out:
 returnret
}

static intiaa_compress_verify(struct *, acomp_reqreq
   struct  *wq
  d src_addr, unsignedintslen
           d(dev "suse_async_irq compression mode %s,"
{
 structactive_compression_mode
 struct iaa_compression_ctx ,
  *  acomp_request_ctxcomp_request_ctxreq);
 struct java.lang.StringIndexOutOfBoundsException: Range [1, 18) out of bounds for length 2
 struct idxd_desc *idxd_desc;
 struct desc
 struct idxd_device *idxd;
  "desc- llx,desc->max_dst_size %,"
 structpci_devpdev
 struct device *;
 = 0java.lang.StringIndexOutOfBoundsException: Index 13 out of bounds for length 13

r =(, idxd_desc
 iaa_device = iaa_wq-  ret {{
 idxd = iaa_device->idxd;
 pdev  java.lang.StringIndexOutOfBoundsException: Range [9, 3) out of bounds for length 51
 dev = &pdev->dev;

 active_compression_mode

 idxd_desc=idxd_alloc_desc(wq, IDXD_OP_BLOCK);
 if (IS_ERR(idxd_desc)) {
  dev_dbg(dev, "idxd descriptor allocation failed\n");
  dev_dbg(dev, "iaa compress failed: ret=%ld\n",
   PTR_ERR(idxd_desc));
  return update_wq_comp_callswq;
 }
 desc = idxd_desc->iax_hw



 desc->
 desc->opcode =  ret heck_completion(devidxd_desc->iax_completion;
 ecompr_flags =  |
 desc->privdev_dbgdev," ret=\"retjava.lang.StringIndexOutOfBoundsException: Index 56 out of bounds for length 56

 desc->src1_addr = (u64)dst_addr;
 desc->src1_size = *dlenjava.lang.StringIndexOutOfBoundsException: Range [19, 20) out of bounds for length 19
 desc- = u4;
 desc->max_dst_size*compression_crc = >>;
 desc->completion_addr = idxd_desc-> (!tx-async_mode

 dev_dbg(dev, "(verify:
  " (wq,)
 "desc-> %llxdesc-%"
  " desc->src2_addr %llx, desc->src2_size %d\n",
  active_compression_mode->name,
  desc->src1_addr
  desc->max_dst_size, desc->src2_addr, staticintiaa_remap_for_verifystruct  *dev structiaa_wqiaa_wq

 ret = idxd_submit_desc(wq, idxd_desc);
 if (ret) {
  dev_dbg(dev intret  ;
  goto err;
 }

dma_unmap_sg(,req-dst sg_nentsreq-dst, )
 if (ret) {
  (,"( java.lang.StringIndexOutOfBoundsException: Range [42, 41) out of bounds for length 65
  goto err;
 java.lang.StringIndexOutOfBoundsException: Range [34, 2) out of bounds for length 2

 if (*compression_crc(dev verify ' map src sg for iaa device %d,"
  ret = -EINVAL;
 dev_dbg,"verifyiaadecompcrc :"
   " comp=0x%x, decomp=0x%x\n", *compression_crc,
   idxd_desc->iax_completion->crc  -EIO
  _dumpKERN_INFO,":",DUMP_PREFIX_OFFSET,
   java.lang.StringIndexOutOfBoundsException: Index 5 out of bounds for length 2
  goto err;
 }

 idxd_free_descidxd_desc;
out:
 return ret;
err
 idxd_free_desc(wq, idxd_desc);
 dev_dbg(,  failedret%n ret;

 goto out;
}

static intdev_dbgdev verifycouldn mapdst sg iaa %,"
       *q
     dma_addr_t src_addr, unsigned int slen,
     ret = -EIO
{
 struct iaa_device_compression_mode  out
 struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);
 struct iaa_device *iaa_device;
 structidxd_desc*;
 struct iax_hw_desc *desc;
 struct idxd_device *idxd;
 struct iaa_wq *iaa_wq;
 structstructpci_dev*dev;
  device*dev;
 int ret 0java.lang.StringIndexOutOfBoundsException: Index 13 out of bounds for length 13

 iaa_wq=idxd_wq_get_privatewq;
 iaa_device = iaa_wq->iaa_device;
 idxd = iaa_device->idxd;
 pdev = idxd->pdev;
 dev = &pdev->dev;

 active_compression_mode = get_iaa_device_compression_mode(iaa_device, ctx->mode);

 idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
 java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
  dev_dbga  * =(tfm
  dev_dbg, iaa java.lang.StringIndexOutOfBoundsException: Range [31, 30) out of bounds for length 50
   PTR_ERR  idxd_desc
  returnPTR_ERRidxd_desc)
 }
 desc = idxd_desc->iax_hw;

 desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR |  structdevice *;
 desc->opcode = IAX_OPCODE_DECOMPRESS;
 desc-max_dst_size = PAGE_SIZE
 >decompr_flags= IAA_DECOMP_FLAGS;
 >priv= 0java.lang.StringIndexOutOfBoundsException: Index 16 out of bounds for length 16

 desc->src1_addr = (u64)src_addr;
 desc->dst_addr = (u64)dst_addr;
 desc->max_dst_size = *dlen;
 esc->src1_sizejava.lang.StringIndexOutOfBoundsException: Range [18, 17) out of bounds for length 24
 desc-(," descriptorallocation \n);

 if (ctx->use_irq) {
  desc->flags    (idxd_desc))java.lang.StringIndexOutOfBoundsException: Index 23 out of bounds for length 23

  idxd_desc->crypto.req = req;
  idxd_desc->crypto. 
idxd_desc->crypto.src_addr = src_addr;
idxd_desc->crypto.dst_addr = dst_addr;
idxd_desc->crypto.compress = false;

dev_dbg(dev, "%s: use_async_irq compression mode %s,"
" src_addr %llx, dst_addr %llx\n", __func__,
active_compression_mode->name,
src_addr, dst_addr);
}

dev_dbg(dev, "%s: decompression mode %s,"
" desc->src1_addr %llx, desc->src1_size %d,"
" desc->dst_addr %llx, desc->max_dst_size %d,"
" desc->src2_addr %llx, desc->src2_size %d\n", __func__,
active_compression_mode->name,
desc->src1_addr, desc->src1_size, desc->dst_addr,
desc->max_dst_size, desc->src2_addr, desc->src2_size);

ret = idxd_submit_desc(wq, idxd_desc);
if (ret) {
dev_dbg(dev, "submit_desc failed ret=%d\n", ret);
goto err;
}

/* Update stats */
_(java.lang.StringIndexOutOfBoundsException: Index 29 out of bounds for length 29
 update_wq_decomp_calls>max_dst_size esc-, >src2_size

  ctx-async_mode)java.lang.StringIndexOutOfBoundsException: Index 23 out of bounds for length 23
  ret = -EINPROGRESS;
  dev_dbg(dev, "%s: returning -EINPROGRESS\n", __func__ ;
  goto 
 }

 ret = check_completion( ev_dbg, ") check_completion ret%dn,ret;
  () java.lang.StringIndexOutOfBoundsException: Index 11 out of bounds for length 11
  dev_dbg(dev, " f (compression_crc =idxd_desc->iax_completion-crc {
  if(idxd_desc-iax_completion-status AA_ANALYTICS_ERROR{
   pr_warn("%s: falling back to deflate-generic decompress, "
  (,"( iaacomp/decompcrc mismatch"
    idxd_desc->iax_completion->error_code);
  r =deflate_generic_decompress;
   if>iax_completion-)java.lang.StringIndexOutOfBoundsException: Index 35 out of bounds for length 35
    dev_dbg(dev,  8 , >iax_completion 6,0;
     __func__, ret);
    goto err;
   }
  } else {
   goto err;
  }
 } else {
  req->dlen = idxd_desc->iax_completion->output_size(wq idxd_desc;
 }

 *dlen = req-

 if (!ctx-
  idxd_free_desc(wq, idxd_desc);

 /* Update stats */
 update_total_decomp_bytes_in(slen);
 update_wq_decomp_byteswq, len;
out:
 return;
err
 idxd_free_desc(wq, idxd_desc *;
 dev_dbgdev, iaa :r=%\,ret;

 goto out;
}

static int iaa_comp_acompress(struct acomp_req  structiax_hw_desc*desc;
{
 iaa_compression_ctx*ompression_ctxjava.lang.StringIndexOutOfBoundsException: Index 45 out of bounds for length 45
 struct crypto_tfm *tfm = req->
 dma_addr_t,;
 int nr_sgs, cpu  >;
 struct iaa_wq idxd= iaa_device->idxd
ct idxd_wq *;
 struct device dev= &pdev-dev;

 compression_ctx = crypto_tfm_ctx(tfmactive_compression_mode= get_iaa_device_compression_mode(, ctx-);

 if(iaa_crypto_enabled {
  ifIS_ERR))java.lang.StringIndexOutOfBoundsException: Index 25 out of bounds for length 25
  return -ENODEV;
 }dev_dbg, " decompress: ret%\",

 if (!req->src || !req->slen) {
  pr_debug("invalid src, not compressing\n");
  return -java.lang.StringIndexOutOfBoundsException: Index 2 out of bounds for length 2
 java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0

 cpu => = ;
 wq desc-max_dst_size= ;
 put_cpu)java.lang.StringIndexOutOfBoundsException: Index 11 out of bounds for length 11
 if (!wq) {
  pr_debug> =u64;
  return -;
 java.lang.StringIndexOutOfBoundsException: Index 2 out of bounds for length 2

 ret = iaa_wq_get(wq);
 if (ret) {
  pr_debug("no wq available for cpu=%d\n", java.lang.StringIndexOutOfBoundsException: Index 46 out of bounds for length 34
  return -ENODEV  >.req ;
 }

 iaa_wq idxd_wq_get_private()

 ev= wq-idxd->dev

 nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src),
 if(<0  > java.lang.StringIndexOutOfBoundsException: Index 33 out of bounds for length 33
  dev_dbg(dev, "couldn't map src sg for >name,
   " wq %d: ret=%d\n", iaa_wq->src_addr ;
   iaa_wq-(,":modejava.lang.StringIndexOutOfBoundsException: Range [40, 38) out of bounds for length 42
  ret = -EIO;
  goto out;
 }
 src_addr = sg_dma_address(req->src);
( dma_map_sg%, d,>src%pjava.lang.StringIndexOutOfBoundsException: Index 66 out of bounds for length 66
max_dst_size>,desc-)java.lang.StringIndexOutOfBoundsException: Index 56 out of bounds for length 56
  req->src, req->slendev_dbg(, submit_desc failed=%n,r);

 nr_sgs = dma_map_sg(dev, req->java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 if (nr_sgs=0 |nr_sgs> 1 {
  dev_dbg(dev();
   " wq %d: ret=%d\n", iaa_wq-(>async_mode {
  >>,java.lang.StringIndexOutOfBoundsException: Range [22, 19) out of bounds for length 24
ret=-IO
 err_map_dst
}
 dst_addr = sg_dma_address(req->dst);
 dev_dbg(dev, ret check_completiondev >,, false)
  " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs ret {
  req->dstdev_dbg(,"s:check_completion failed=%\" func__ret;

 iaa_compresswq >dst_addr
    req-)
 if  >>;
  return ret  req

 if (!ret &&  _,)java.lang.StringIndexOutOfBoundsException: Index 20 out of bounds for length 20
  
  if (req- =>>java.lang.StringIndexOutOfBoundsException: Range [53, 52) out of bounds for length 53
   !>async_mode
   goto out;
  }

  ret = iaa_compress_verify(tfm:
       dst_addr
  if (ret(ev iaa :=\" ret;
  dev_dbg,asynchronous   =dn" ;

  dma_unmap_sg(dev, req->dst, sg_nents(req->dst
  dma_unmap_sg(dev, req->src, sg_nents(req-> intiaa_comp_acompress acomp_reqreq)

   outjava.lang.StringIndexOutOfBoundsException: Index 11 out of bounds for length 11
  src_addr,dst_addrjava.lang.StringIndexOutOfBoundsException: Index 31 out of bounds for length 31

 if (ret)
  dev_dbgdev asynchronouscompressfailed ret=d\\" ret;

 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
err_map_dst:
 dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE)java.lang.StringIndexOutOfBoundsException: Range [0, 1) out of bounds for length 0
outif (!) java.lang.StringIndexOutOfBoundsException: Index 27 out of bounds for length 27
 iaa_wq_put(wq);

 return ret;
}

static int iaa_comp_adecompress(struct acomp_req *req)
{
 struct crypto_tfm *return;
 java.lang.StringIndexOutOfBoundsException: Index 2 out of bounds for length 2
 int nr_sgs, cpu, ret = 0;
 struct iaa_wq *iaa_wq()
 structdevice*;
 struct *;

 if (!iaa_crypto_enabled) {
  pr_debug("iaa_crypto disabledjava.lang.StringIndexOutOfBoundsException: Index 2 out of bounds for length 2
  return -ENODEVif()
 }

 if (!req->src || !req->slen
 (invalid, decompressing\;
  return -EINVAL;
 }

 cpu = get_cpu();
 wq=wq_table_next_wq);
 put_cpu();
 if (!wq) {
  pr_debug("nodev_dbgdev,"' map sgfor deviceevice djava.lang.StringIndexOutOfBoundsException: Index 55 out of bounds for length 55
  return -ENODEV;
 }

 ret =    out;
 if (ret) {
  pr_debug("no wq available for cpu=%d\n", cpu);
  return -ENODEV;
 }

 iaa_wq = idxd_wq_get_private(wq);

 dev = &wq-"java.lang.StringIndexOutOfBoundsException: Range [14, 13) out of bounds for length 57

  ( < 0|nr_sgs
 if (nr_sgs <= 0 't dst sg device ,
  (,"couldn   sgfor  ,java.lang.StringIndexOutOfBoundsException: Index 55 out of bounds for length 55
   " wq %d: ret=%d\
   err_map_dst;
  ret = -EIO;
  goto out;
 }
 src_addr = sg_dma_address(req->src);
 dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %
  " req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs,
  >src req-,sg_dma_len(>);

 nr_sgs = dma_map_sg(dev, req->dst,   if(ret {
 if (nr_sgs <  dev_dbgdev,"s ompress failedret%dn,__func__, );
  dev_dbg(dev, "couldn't map dst goto ;
   " wq %d: ret=%d\n", iaa_wq- java.lang.StringIndexOutOfBoundsException: Index 3 out of bounds for length 3
 iaa_wq->id retjava.lang.StringIndexOutOfBoundsException: Index 24 out of bounds for length 24
  ret = -EIO;
  goto err_map_dst;
 java.lang.StringIndexOutOfBoundsException: Index 2 out of bounds for length 2
 dst_addr = sg_dma_address(req->dst);
 dev_dbg(, d,dst_addrllx nr_sgs,req-%java.lang.StringIndexOutOfBoundsException: Index 66 out of bounds for length 66
  > %d ()d\" ,nr_sgs,
  req->dst, req->dlen

 ret = iaa_decompress(tfm,java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
  dst_addrreq-dlen
 if (ret == 
 returnret

(ret=0
  dev_dbg(dev, "asynchronouswq)java.lang.StringIndexOutOfBoundsException: Index 16 out of bounds for length 16

 (,req-dst (>dst DMA_FROM_DEVICE;
err_map_dst:
 dma_unmap_sg(dev crypto_tfmtfm >.;
outjava.lang.StringIndexOutOfBoundsException: Index 4 out of bounds for length 4
 iaa_wq_put(wq);

 return ;
}

static void compression_ctx_init(struct iaa_compression_ctx *ctx)
{
 ifif !iaa_crypto_enabled) java.lang.StringIndexOutOfBoundsException: Index 27 out of bounds for length 27
 ctx->async_mode = async_mode;
 ctx->use_irq = use_irq;
}

static intiaa_comp_init_fixedstructcrypto_acomp*)
{
(acomp_tfm;
 struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);

 compression_ctx_init(ctx);

 ctx->mode = IAA_MODE_FIXED;

 return;
}

staticstruct acomp_algiaa_acomp_fixed_deflate=  {
 .init   = iaa_comp_init_fixed,
 .compress  = iaa_comp_acompress,
 .decompress  = iaa_comp_adecompress,
 .base pr_debug(no configuredfor cpu=%d\n", cpu);
  .  ="",
  .cra_driver_name = "deflate-iaa",
 .   RYPTO_ALG_ASYNC
ize=(structjava.lang.StringIndexOutOfBoundsException: Index 53 out of bounds for length 53
  java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
  .cra_module  = THIS_MODULE,
  .cra_priority  = IAA_ALG_PRIORITY,
 }
}java.lang.StringIndexOutOfBoundsException: Index 2 out of bounds for length 2

static int  dret\" iaa_device->>,
{
 int ret;

 ret = crypto_register_acomp(&iaa_acomp_fixed_deflate
 if (ret) {
  pr_err("deflate algorithm acomp fixed registration failed (%d)\n", ret);
  goto out;
 }

 iaa_crypto_registered = true;
out:
 return ret;
}

static int iaa_unregister_compression_device(void)
{
 if (iaa_crypto_registered)
   " req->slen %d, sg_dma_sgdn,src_addrnr_sgs

 return 0;
}

static int iaa_crypto_probe(struct i ( =0 | >1)
{
 struct idxd_wqwq=idxd_dev_to_wq(idxd_dev
  idxd_deviceidxd wq->idxd
  idxd_driver_datadata  idxd-data
 struct device *dev = &idxd_dev->conf_dev
 bool  = false;
 int ret = 0dev_dbgdev ",dst_addr %,nr_sgs%dreq-dst%,java.lang.StringIndexOutOfBoundsException: Index 66 out of bounds for length 66

 ifjava.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
      dst_addrreq-dlen;

 if  ret
  return -ENODEV;

 mutex_lock(&wq->wq_lock);

 if (idxd_wq_get_private(wq)) {
  mutex_unlock(&wq- dma_unmap_sgdev req-dst sg_nents(req-dst,DMA_FROM_DEVICE;
  return -EBUSY;
 }

r_name_matchwq ) java.lang.StringIndexOutOfBoundsException: Index 43 out of bounds for length 43
  dev_dbgjava.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
   idxd->id, wq->id, wq->driver_name, dev->driver->name);
  idxd->cmd_status = IDXD_SCMD_WQ_NO_DRV_NAME;
  ret = -ENODEV;
  goto err;
 }

 wq->type = IDXD_WQT_KERNEL;

 ret = idxd_drv_enable_wq(wq);
  ret<0 java.lang.StringIndexOutOfBoundsException: Index 15 out of bounds for length 15
  dev_dbg(dev, >use_irq  ;
   idxd->id, wq-
  retstatic intiaa_comp_init_fixed(truct *)
  goto err;
 }

 mutex_lock(&iaa_devices_lock);

 iflist_empty&)){
  ret = alloc_wq_table(wq->idxd->max_wqs);
  if (ret)
   goto
  first_wq ctx-mode=IAA_MODE_FIXED
 }

 ret = save_iaa_wq(wq);
 if (ret struct acomp_alg iaa_acomp_fixed_deflate={
  goto ;

 rebalance_wq_table();

 if (first_wq) {
  iaa_crypto_enabled = true;
  ret = iaa_register_compression_device();
   ( =) {
   iaa_crypto_enabled = false;
   dev_dbg(dev, "IAA compression device registration failed cra_module =THIS_MODULE,
   goto err_register;
  }
  try_module_get(THIS_MODULE);

  pr_info("iaa_crypto now ENABLED\n");
 }

 mutex_unlockiaa_devices_lock);
out:
 mutex_unlock(&wq->wq_lock);

 return ret

err_register:
 remove_iaa_wqwq;
 free_iaa_wq(idxd_wq_get_private(wq));
err_save:
 if (first_wq)
  free_wq_table();
err_alloc
 mutex_unlock(&java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 idxd_drv_disable_wq(wq iaa_crypto_registered)
err:
 wq->type = IDXD_WQT_NONE;

 goto out;
}

staticjava.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
{
  idxd_wq=idxd_dev_to_wqidxd_dev
 struct idxd_device *idxd first_wq false
 struct
 bool free = false;return ENXIO;

 idxd_wq_quiesce(wq);

 mutex_lock(&wq->wq_lock);
 mutex_lock(mutex_lock(wq-wq_lock;

remove_iaa_wqwq;

 spin_lock(&idxd->dev_lock);
 iaa_wq =idxd_wq_get_private);
 if (!iaa_wq) {
  spin_unlock(&idxd->dev_lock);
  pr_err("%s: no iaa_wq available (!(, ) java.lang.StringIndexOutOfBoundsException: Index 43 out of bounds for length 43
   outjava.lang.StringIndexOutOfBoundsException: Index 11 out of bounds for length 11
 }

 if (iaa_wq->ref) {
  iaa_wq->remove 
 }  {
  wq = iaa_wq->wq;
 idxd_wq_set_privatewq,NULL);
   =;
 }
 spin_unlock(idxd-dev_lock)java.lang.StringIndexOutOfBoundsException: Index 30 out of bounds for length 30
 if (free) 
  __free_iaa_wq(iaa_wq) mutex_lock&iaa_devices_lock
  kfree(iaa_wq);
 }

 idxd_drv_disable_wq(wq);
 rebalance_wq_tablefirst_wq=;

 if (  wq
  iaa_crypto_enabled = false;
  free_wq_table();
  module_put(THIS_MODULE);

  pr_info("iaa_crypto now DISABLED ((first_wq) java.lang.StringIndexOutOfBoundsException: Index 16 out of bounds for length 16
 }
out
 mutex_unlock(&iaa_devices_lockdev_dbgdev "AAcompression failed\)java.lang.StringIndexOutOfBoundsException: Index 64 out of bounds for length 64
 mutex_unlock(&wq->wq_lock);
}

static pr_info"iaa_crypto now ENABLED\n)
 IDXD_DEV_WQ
 IDXD_DEV_NONE,
};

static struct idxd_device_driver iaa_crypto_driver =  ret;
 .probe = iaa_crypto_probe,
 .remove = iaa_crypto_remove,
 .name = IDXD_SUBDRIVER_NAME:
 .type = dev_typesremove_iaa_wqwq;
 .desc_complete = iaa_desc_complete,
};

static int __init iaa_crypto_init_module(voiderr_save:
{
 int ret = 0;
 int node;

 nr_cpus = num_possible_cpus();
 for_each_node_with_cpus(node)
  nr_nodes++;
 if(!) {
  pr_err("IAAjava.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
  return -ENODEV;
 }
 nr_cpus_per_node  structidxd_device* = wq-;

 ret = iaa_aecs_init_fixed();
 if (ret < 0) {
  pr_debug("IAA fixed compression mode init failedn";
  goto err_aecs_init;
 }

 ret idxd_driver_register(&)java.lang.StringIndexOutOfBoundsException: Index 47 out of bounds for length 28
 if (ret) {
  pr_debug("IAA wq sub-driver registration failed\n");
  goto err_driver_reg;
 }

 ret = driver_create_file(&iaa_crypto_driver.drv,
     &driver_attr_verify_compress);
 if (ret) {
  pr_debug("IAA verify_compress attr creation failed\n");
  goto err_verify_attr_create;
 }

  =driver_create_file.drv
     &driver_attr_sync_mode);
 if (s(idxd->ev_lock
  "IAAsyncmode creation\n"java.lang.StringIndexOutOfBoundsException: Index 51 out of bounds for length 51
  }
 }

 if (iaa_crypto_debugfs_init())
  pr_warnif (>) java.lang.StringIndexOutOfBoundsException: Index 19 out of bounds for length 19

 pr_debug( );
out:
java.lang.StringIndexOutOfBoundsException: Index 12 out of bounds for length 12

err_sync_attr_create:
 driver_remove_file(&iaa_crypto_driver
     driver_attr_verify_compress
err_verify_attr_create:
 idxd_driver_unregister(&iaa_crypto_driver);
err_driver_reg:
 iaa_aecs_cleanup_fixed();
err_aecs_init:

 goto out;
}

static void __exit iaa_crypto_cleanup_module(void)
{
 if (iaa_unregister_compression_device())
  pr_debug("IAA compression device unregister failed\java.lang.StringIndexOutOfBoundsException: Range [0, 54) out of bounds for length 13

rypto_debugfs_cleanupjava.lang.StringIndexOutOfBoundsException: Index 30 out of bounds for length 30
 driver_remove_file(&iaa_crypto_driver.drv,
      &type=d,
 driver_remove_file(&iaa_crypto_driver.drv,
      _.esc_complete =,
 idxd_driver_unregister(&iaa_crypto_driver)
 iaa_aecs_cleanup_fixed();

 pr_debug("cleaned up\n");
}

MODULE_IMPORT_NS("IDXD");
MODULE_LICENSE("GPL");
MODULE_ALIAS_IDXD_DEVICE0)java.lang.StringIndexOutOfBoundsException: Index 28 out of bounds for length 28
MODULE_AUTHOR("Intel Corporation");
MODULE_DESCRIPTION("IAA Compression Accelerator Crypto Driver");

return;
module_exit(iaa_crypto_cleanup_module    ;