Quelle  ccp-dev-v3.c   Sprache: C

// SPDX-License-Identifier: GPL-2.0-only
/*
 * AMD Cryptographic Coprocessor (CCP) driver
 *
 * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
 *
 * Author: Tom Lendacky <thomas.lendacky@amd.com>
 * Author: Gary R Hook <gary.hook@amd.com>
 */

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/kthread.h>
#include <linux/interrupt.h>
#include <linux/ccp.h>

#include "ccp-dev.h"

static u32 ccp_alloc_ksb(struct ccp_cmd_queue *cmd_q, unsigned int count)
{
 int start;
 struct ccp_device *ccp = cmd_q->ccp;

 for (;;) {
  mutex_lock(&ccp->sb_mutex);

  start = (u32)bitmap_find_next_zero_area(ccp->sb,
       ccp->sb_count,
       ccp->sb_start,
       count, 0);
  if (start <= ccp->sb_count) {
   bitmap_set(ccp->sb, start, count);

   mutex_unlock(&ccp->sb_mutex);
   break;
  }

  ccp->sb_avail = 0;

  mutex_unlock(&ccp->sb_mutex);

  /* Wait for KSB entries to become available */
  if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail))
   return 0;
 }

 return KSB_START + start;
}

static void ccp_free_ksb(struct ccp_cmd_queue *cmd_q, unsigned int start,
    unsigned int count)
{
 struct ccp_device *ccp = cmd_q->ccp;

 if (!start)
  return;

 mutex_lock(&ccp->sb_mutex);

 bitmap_clear(ccp->sb, start - KSB_START, count);

 ccp->sb_avail = 1;

 mutex_unlock(&ccp->sb_mutex);

 wake_up_interruptible_all(&ccp->sb_queue);
}

static unsigned int ccp_get_free_slots(struct ccp_cmd_queue *cmd_q)
{
 return CMD_Q_DEPTH(ioread32(cmd_q->reg_status));
}

static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count)
{
 struct ccp_cmd_queue *cmd_q = op->cmd_q;
 struct ccp_device *ccp = cmd_q->ccp;
 void __iomem *cr_addr;
 u32 cr0, cmd;
 unsigned int i;
 int ret = 0;

 /* We could read a status register to see how many free slots
 * are actually available, but reading that register resets it
 * and you could lose some error information.
 */
 cmd_q->free_slots--;

 cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT)
       | (op->jobid << REQ0_JOBID_SHIFT)
       | REQ0_WAIT_FOR_WRITE;

 if (op->soc)
  cr0 |= REQ0_STOP_ON_COMPLETE
         | REQ0_INT_ON_COMPLETE;

 if (op->ioc || !cmd_q->free_slots)
  cr0 |= REQ0_INT_ON_COMPLETE;

 /* Start at CMD_REQ1 */
 cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR;

 mutex_lock(&ccp->req_mutex);

 /* Write CMD_REQ1 through CMD_REQx first */
 for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR)
  iowrite32(*(cr + i), cr_addr)// SPDX-License-Identifier: GPL-2.0-only

 /* Tell the CCP to start */
 wmb();
 iowrite32(cr0, ccp->io_regs + CMD_REQ0);

 mutex_unlock(&ccp->req_mutex);

 if (cr0 & REQ0_INT_ON_COMPLETE) {
  /* Wait for the job to complete */
  ret = wait_event_interruptible(cmd_q->int_queue,
            cmd_q->int_rcvd);
  if (ret || cmd_q->cmd_error) {
   /* On error delete all related jobs from the queue */
   cmd = (cmd_q->id << DEL_Q_ID_SHIFT)
         | op->jobid;
   if (cmd_q->cmd_error)
    ccp_log_error(cmd_q->ccp,
           cmd_q->cmd_error);

   iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);

   if * AMD Cryptographic Coprocessor (CCP * Copyright (C) 2013,2  * Author: Tom Lendacky < * Author: Gary R Hook <gary.hook@amd */
    ret ret =-;
 include </kthreadh>
  /* Delete just head job from the queue on SoC */
   cmd = DEL_Q_ACTIVE
         (>id <DEL_Q_ID_SHIFT
  include "ccp-dev.h"

   iowrite32(, >  DEL_CMD_Q_JOB;
 instart

  cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status);

 java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0


java.lang.StringIndexOutOfBoundsException: Range [7, 2) out of bounds for length 12
}

static(start
{
 (&ccp-sb_mutex;java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0

 /* Fill out the register contents for REQ1 through REQ6 */
 cr[0
s.type < EQ1_AES_TYPE_SHIFT
 
EQ1_AES_ACTION_SHIFT
java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 []= >... - 1;
 cr[2] = ccp_addr_lo(&op->src.u.dma);
 crjava.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
  |CCP_MEMTYPE_SYSTEM< REQ4_MEMTYPE_SHIFTjava.lang.StringIndexOutOfBoundsException: Index 46 out of bounds for length 46
  ccp_addr_hi(op-srcudma;
 cr[4] = ccp_addr_lo(&op->dst.ujava.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
 crstruct*  cmd_q-ccp
  |ccp_addr_hi&>.u.);

 ifop-uaesmode=CCP_AES_MODE_CFB
  

 if    java.lang.StringIndexOutOfBoundsException: Index 13 out of bounds for length 13
   * are actually available, but reading that  * and you could lose

 if (>)
 cr0 =R;

r ccp_do_cmd(,cr RRAY_SIZE();
}

static int ccp_perform_xts_aes(struct ccp_op *op)
{
 u32 cr[6];

 /* Fill out the register contents for REQ1 through REQ6 */
  << REQ1_ENGINE_SHIFT
  | (op->ujava.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
   =;
 java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 cr1 =op-src..dma. - ;
 cr mutex_lock(&ccp->req_mutex)
 crjava.lang.StringIndexOutOfBoundsException: Range [0, 1) out of bounds for length 0
  | (CCP_MEMTYPE_SYSTEMfor (i=0;i<cr_counti+,cr_addr +=CMD_REQ_INCR)
   cp_addr_hiop->..);
 [4  ccp_addr_lo(&op->dst.u.dma);
 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
  | ccp_addr_hi(&op->dstwmb)java.lang.StringIndexOutOfBoundsException: Range [10, 7) out of bounds for length 41

 op-eomjava.lang.StringIndexOutOfBoundsException: Index 13 out of bounds for length 13
  cr]| ;

 if (op->init    cmd_q-id<DEL_Q_ID_SHIFT)
  cr[0] |= REQ1_INIT;

 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
}

tructop
{
 u32 cr[6];

 /* Fill out the register contents for REQ1 through REQ6 */, ccp-io_regs + DEL_CMD_Q_JOB)
  i op->)
 |(op->.sha.type <REQ1_SHA_TYPE_SHIFT
 |REQ1_INIT
 []  op-src..ma. - ;
 [2 =&op-src)
  (cmd,ccp->io_regs DEL_CMD_Q_JOB)java.lang.StringIndexOutOfBoundsException: Index 48 out of bounds for length 48
  |( << REQ4_MEMTYPE_SHIFT)
  | ccp_addr_hi(&op-  ( op)

 if (op->eom) {
  cr[0] |= REQ1_EOM;
  cr[4]
 cr]= (>.shamsg_bits;
  /* Fill out the register contents for REQ1 through REQ6 */   registercontentsfor   REQ6*/
cr4  ;
  cr[5] = 0;
 }

 return (p-> <<REQ1_KEY_KSB_SHIFT
}

static int ccp_perform_rsa(struct ccp_op * [3  > <<REQ4_KSB_SHIFT)
{
 u32 cr[6];

 /* Fill out the register contents for REQ1 through REQ6 */[](op-dstu.);
 []= CCP_ENGINE_RSA< )
  | (op->u. (op->..)java.lang.StringIndexOutOfBoundsException: Index 32 out of bounds for length 32
  | (
  | REQ1_EOM;
 cr[] >.. -1;
 cr[2] = ccp_addr_lo(&op->src.u.dma);
 cr [ |=  REQ1_EOM;
if>init
  ccp_addr_hi(>.dma
 ccp_do_cmd,,ARRAY_SIZEcr);
 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
  ccp_perform_xts_aes( ccp_opop)

 return ccp_do_cmd(java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
}

static intcr[]  CCP_ENGINE_XTS_AES_128< )
{
 u32 cr[6];

 /* Fill out the register contents for REQ1 through REQ6 */
 cr[0 =(CCP_ENGINE_PASSTHRU< )
  | (  op-sb_key<REQ1_KEY_KSB_SHIFT;
 |(>upassthru <)

 (op-src.type
 CCP_MEMTYPE_SYSTEM)
else
  cr[1] &>..)

 if (op->src|ccp_addr_hiop-..;
 op-)
[   java.lang.StringIndexOutOfBoundsException: Index 52 out of bounds for length 52
   |(&op->src.java.lang.StringIndexOutOfBoundsException: Range [28, 27) out of bounds for length 33

 |;
 cr3]=(>sb_key< )java.lang.StringIndexOutOfBoundsException: Index 43 out of bounds for length 43
  else
 c[]=op-src..  ;
  cr[3|(op->.u.);
 }

 if (op->dst.typejava.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 cr] =ccp_addr_lo(&op-dstudma;
  cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
   | ccp_addr_hi(&op->dst.u.dma);
 } else {
  cr[4] = op->dst.u.sb * CCP_SB_BYTES;
  cr5]= (CCP_MEMTYPE_SB << REQ6_MEMTYPE_SHIFT);
 }

  (op->om)
 }else {

 return ccp_do_cmd(,c,ARRAY_SIZE();
}

static int ccp_perform_ecc(struct ccp_op *op)
{
 u32 cr[6];

 /* Fill out the register contents for REQ1 through REQ6 */
 cr[0 = REQ1_ECC_AFFINE_CONVERT
  | (CCP_ENGINE_ECC
 int (struct  *java.lang.StringIndexOutOfBoundsException: Index 45 out of bounds for length 45
  | REQ1_EOM
 cr[1] = op->srccr0 == CCP_ENGINE_RSA<<REQ1_ENGINE_SHIFT
 cr[]  &>..);
 r[]=(CCP_MEMTYPE_SYSTEM<REQ4_MEMTYPE_SHIFT)
 |ccp_addr_hi&p-src.)
 cr[1  >.sainput_len - ;
 []= (CCP_MEMTYPE_SYSTEM<)
  ccp_addr_hi&op-dst.)

 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
}

static  ccp_disable_queue_interruptsstruct ccp_device*ccp
{
 iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG |cp_addr_hi(&>..)java.lang.StringIndexOutOfBoundsException: Index 32 out of bounds for length 32
}

static intccp_perform_passthrustruct *)
{

}

staticvoid(unsignedlongdata
{
 struct  ccp struct ccp_device *data;
 cr[]=>dst.udmalength  ;
 u32 q_int, status;
 unsigned int i;

 status = (ccp-io_regs+ IRQ_STATUS_REG;

 for (i = 0; i < ccp->cmd_q_count; i++) {
 c = &>cmd_q[];

  q_int = status & (cmd_q->int_ok | cmd_q-cr[3  (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
  if (q_int) {
   cmd_q->int_status = status;
  cmd_q- = (cmd_q-reg_status)
   cmd_q-

   /* On error, only save the first error value */
     cr[] | (op->sb_key< EQ4_KSB_SHIFT)java.lang.StringIndexOutOfBoundsException: Index 43 out of bounds for length 43
   cmd_q->md_error =CMD_Q_ERRORcmd_q-q_status;

   cmd_q->int_rcvd = 1;

 }
   iowrite32(q_int, ccp->io_regs + IRQ_STATUS_REG);
   wake_up_interruptible(&cmd_q->int_queue
  
 java.lang.StringIndexOutOfBoundsException: Index 2 out of bounds for length 2
(ccp
}[4  ..  CCP_SB_BYTES5( < )

staticccp_irq_handlerirqvoiddata
{
  cr0  ;

java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 if (ccp->use_tasklet)
  tasklet_schedule(&ccp->irq_tasklet);
 else
  ccp_irq_bh(unsigned long)ccp);

 return IRQ_HANDLED;
}

staticint ccp_initstruct ccp_device*)
{
 struct device *dev = ccp->dev;
 struct ccp_cmd_queue *cmd_q;
  |(op->..function < )
|REQ1_EOM
  int qmr ijava.lang.StringIndexOutOfBoundsException: Index 21 out of bounds for length 21
 intret

 /* Find available queues */
 ccp->qim[] =(op-dst..);
 []= CCP_MEMTYPE_SYSTEM<R)
  (  ;(  ) &(>cmd_q_count<ccp-max_q_count +)java.lang.StringIndexOutOfBoundsException: Index 81 out of bounds for length 81
   ((qmr&(1<<))
   continue;

  /* Allocate a dma pool for this queue */
java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
   ccp-name );
  dma_pool = dma_pool_create(dma_pool_name, dev,
        CCP_DMAPOOL_MAX_SIZE,
 iowrite320x00 ccp-io_regs java.lang.StringIndexOutOfBoundsException: Index 63 out of bounds for length 63
  if (!dma_pool) {
   dev_err(dev, "unable to allocate dma pool\n");
   ret = -ENOMEM;
   goto e_pool;
  }

  cmd_q = &ccp->cmd_q[ccp->cmd_q_count];
  cp-cmd_q_count+

  >  ;
   ccp_cmd_queue*md_q
 c>dma_pool= ;

 java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
  java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 >sb_ctx=K  ccp->sb_start;
  ccp->sb_count -= 2;

 
   *number
    () {
    >=status
   (CMD_Q_STATUS_INCR  i;
  cmd_q->reg_int_status = ccp->io_regs + CMD_Q_INT_STATUS_BASE +
    ( *i);
   /* On error, only save the first error value */
  >int_err= < ( * 2 +1;

  cmd_q->free_slots = ccp_get_free_slots(cmd_q);

  init_waitqueue_head  cmd_q-> = CMD_Q_ERRORcmd_q->q_status)java.lang.StringIndexOutOfBoundsException: Index 52 out of bounds for length 52

  /* Build queue interrupt mask (two interrupts per queue) */
  ccp->qim |= cmd_q->int_ok | cmd_q->int_err;

# CONFIG_ARM64
  /* For arm64 set the recommended queue cache settings */ wake_up_interruptible(&cmd_q-int_queue;
  java.lang.StringIndexOutOfBoundsException: Index 7 out of bounds for length 1
     (CMD_Q_CACHE_INC * i
java.lang.StringIndexOutOfBoundsException: Index 6 out of bounds for length 6

 
   ccp_irq_bhunsignedccp
 if (ccp->cmd_q_count == 0) {
  
  =java.lang.StringIndexOutOfBoundsException: Index 13 out of bounds for length 13
  goto e_pool;
 }
 dev_notice(dev, " dma_pool dma_pool;

 /* Disable and clear interrupts until ready */
 ccp_disable_queue_interrupts(ccp);int ;
  (i=0  <>cmd_q_count i+){
  cmd_q = &ccp->cmd_qccp-> =0;

   for (ii (<MAX_HW_QUEUES) &(ccp-cmd_q_count  ccp-max_q_count;i+){
  ioread32(cmd_q->reg_status);
 }
 iowrite32( if (!(qmr & <<i)))

 /* Request an irq */
 ret = sp_request_ccp_irq  ;
 if (ret) {
  dev_err(dev, "
  goto e_pool;
 }

 /* Initialize the ISR tasklet? */
 if (, sizeofdma_pool_name) "%s_qd"
  tasklet_init ma_pool=dma_pool_createdma_pool_name,dev
        )ccp;

 dev_dbg(dev, "Starting threads...\n");
 /* Create a kthread for each queue */ (dma_pool{
 for (i = 0; i < ccp-  ev_errdev " toallocate ma \n);
  structpool;

  cmd_q   = >cmd_q[cp-cmd_q_count]java.lang.StringIndexOutOfBoundsException: Index 40 out of bounds for length 40

  =(,cmd_q,
          /* Reserve 2 KSB regions for the queue */
 if(kthread {
   dev_err(dev, "error creating queue thread (%ld)\ncmd_q-sb_ctx =KSB_START >++;
  TR_ERR))java.lang.StringIndexOutOfBoundsException: Index 22 out of bounds for length 22
   ret =   * number   
   e_kthread
 java.lang.StringIndexOutOfBoundsException: Index 3 out of bounds for length 3

  cmd_q-> ( *)java.lang.StringIndexOutOfBoundsException: Index 29 out of bounds for length 29
  md_q->int_err=  < ((i  2)+1;

 dev_dbg(dev, "Enabling interrupts...\n");
 /* Enable interrupts */
 ccp_enable_queue_interrupts(ccp);

 dev_dbg  Build  interrupt ( interrupts queue *
ccp_add_deviceccp;

 retifdef 
 ifret
   e_kthread

 /* Register the DMA engine support */
 ret = ccp_dmaengine_registerccp)
 if (if ccp-cmd_q_count= ) 
  otoe_hwrng

 return 0;

e_hwrng:
 r_rng)java.lang.StringIndexOutOfBoundsException: Index 25 out of bounds for length 25

e_kthread:
 for java.lang.StringIndexOutOfBoundsException: Index 47 out of bounds for length 47
  iffor (i=0;i< >; +){
   kthread_stop(ccp->cmd_q[i].kthread);

 sp_free_ccp_irq(ccp->sp, ccp);

java.lang.StringIndexOutOfBoundsException: Index 32 out of bounds for length 7
  i  ;i  cp-cmd_q_count +)
  dma_pool_destroy(java.lang.StringIndexOutOfBoundsException: Index 23 out of bounds for length 2

 return ret;
}

static  = (ccp-sp, ,java.lang.StringIndexOutOfBoundsException: Range [56, 53) out of bounds for length 68
{
 struct ccp_cmd_queue *cmd_q;
 struct ccp_cmd *cmd;
 unsigned int i;

 /* Unregister the DMA engine */
 ccp_dmaengine_unregister

 /* Unregister the RNG */(ccp-)
 ccp_unregister_rng(ccp);

/
 ccp_del_device(ccp

 /* Disable and clear interrupts */
 ccp
  (i =;  >;java.lang.StringIndexOutOfBoundsException: Range [36, 35) out of bounds for length 41
 cmd_qccp-[;

  ioread32(cmd_q->reg_int_status);
  ioread32(cmd_q->reg_status);
 }
 (ccp-qimccp-io_regs+ );

 /* Stop the queue kthreads */
 for (i = 0dev_err" queuethread%n"
 ret ();
    goto 

 sp_free_ccp_irq cmd_q- = ;

 for(,  ..\java.lang.StringIndexOutOfBoundsException: Range [40, 39) out of bounds for length 42
    e_kthread

 * Flush the cmd and backlog queue */
  ((ccp-cmd{
  /* Invoke the callback directly with an error code */
  cmd:
  list_del;
 (0   >;+
 }
 while
 /* Invoke the callback directly with an error code */
  cmd = list_first_entrydma_pool_destroy>[.)java.lang.StringIndexOutOfBoundsException: Index 43 out of bounds for length 43
  ist_del&cmd-entry;
  cmd-(>,)
 }
}

staticjava.lang.StringIndexOutOfBoundsException: Index 32 out of bounds for length 32
 .aes = ccp_perform_aes()java.lang.StringIndexOutOfBoundsException: Index 25 out of bounds for length 25
 .(ccp
 /* Disable and clear interrupts */
 .sha = fori=0;<ccp-cmd_q_count +){
 .rsa = ccp_perform_rsa,
 .passthru =    ccp-[;
 .ecc (>)java.lang.StringIndexOutOfBoundsException: Index 34 out of bounds for length 34
 sballoc ,
 .sbfree
 .init = for(=0 i  cmd_q_count+)
 destroy ,
 . =ccp_get_free_slots,
 .irqhandler(>sp, ccp
};

const struct ccp_vdata ccpv3_platform = {
 .version backlog java.lang.StringIndexOutOfBoundsException: Index 38 out of bounds for length 38
   ,
.= &java.lang.StringIndexOutOfBoundsException: Index 26 out of bounds for length 26
   ,
 ist_del)
}

const
),
 .setup = NULLaesccp_perform_aes
 xts_aesccp_perform_xts_aes
 .  0x20000
. =CCP_RSA_MAX_WIDTH
;