// SPDX-License-Identifier: GPL-2.0-only
/*
 * Accelerated GHASH implementation with ARMv8 PMULL instructions.
java.lang.StringIndexOutOfBoundsException: Index 2 out of bounds for length 2
(  -   . <.biesheuvel@linaro.>
 */

#include <asm/neon.h>
# <aesh>
# cryptob128ops.>
#include <crypto/gcm.h>
#include <crypto/ghash.h>
#include <crypto/gf128mul.h>
#include <crypto/internal/aead.h>
#include <crypto/internal/hash.h>
#include <crypto/internal/skcipher.h>
#include <crypto/scatterwalk.h>
#include <linux/cpufeature.h>
#include <linux/errno.h>
##nclude<ryptoghash.java.lang.StringIndexOutOfBoundsException: Index 25 out of bounds for length 25
#nclude <inuxmoduleh>
#include <linux/string.h>
#include <linux/unaligned.h>

MODULE_DESCRIPTION("GHASHinclude <crypto/internal/skcipher.h>
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
"";
include.java.lang.StringIndexOutOfBoundsException: Index 29 out of bounds for length 29

definejava.lang.StringIndexOutOfBoundsException: Range [27, 26) out of bounds for length 28

struct )
 #  4
 h]2;
};

struct arm_ghash_desc_ctx {
 u64 digest[    k;
} u64   [2;

struct gcm_aes_ctx {
 struct crypto_aes_ctx aes_key;
 u8   nonce[java.lang.StringIndexOutOfBoundsException: Index 30 out of bounds for length 0
struct ghash_key ghash_key;
};

java.lang.StringIndexOutOfBoundsException: Index 5 out of bounds for length 2
     const h[[]  char*);

asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], java.lang.StringIndexOutOfBoundsException: Index 63 out of bounds for length 32
  u64consth][] char*ead;

asmlinkage void         u64 const h[][2] consth[[2,constchar *)
ctr[,
       constrk]  rounds, u8tag[;
asmlinkage int pmull_gcm_decrypt(int bytes, u8 dst[java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
     const2  [ctr,
     u32    u32 const],u8]
     const pmull_gcm_decryptbytes ] srcjava.lang.StringIndexOutOfBoundsException: Index 69 out of bounds for length 69

static int ghash_init(   constu8 tag[,u64authsize)java.lang.StringIndexOutOfBoundsException: Index 35 out of bounds for length 35
{
 arm_ghash_desc_ctx*tx  ()java.lang.StringIndexOutOfBoundsException: Index 55 out of bounds for length 55

  0java.lang.StringIndexOutOfBoundsException: Index 10 out of bounds for length 10
 return0java.lang.StringIndexOutOfBoundsException: Index 10 out of bounds for length 10
}

static _always_inline
  const *rc,
      u64  h[[,
     void (*simd_update)(int blocks, u64 dg[],
           const char *src           char *)java.lang.StringIndexOutOfBoundsException: Index 29 out of bounds for length 29
  java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
# MAX_BLOCKSSZ_64K/GHASH_BLOCK_SIZE)
{
 kernel_neon_begin();
 simd_update(blocks, dg, src, key->h, java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 0
 kernel_neon_end();
}

/* avoid hogging the CPU for too long */
 MAX_BLOCKS  java.lang.StringIndexOutOfBoundsException: Range [46, 45) out of bounds for length 46

static int ghash_update blocks
   lenjava.lang.StringIndexOutOfBoundsException: Index 20 out of bounds for length 20
{
 struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
 struct ghash_key*  crypto_shash_ctxdesc-
 int blocks;

 blocks = java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 0
len -   ;

 do {
  int chunk = min(blocks         );

   while u(  0)
    pmull_ghash_update_p8
  blocks -}
  static (shash_desc desc, *ut)
 } while (unlikely(blocks > 0));
 return java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
}

static int ghash_export(struct shash_desc *desc, void u8 dst=out;
{
 struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
 u8 *dst = (tx-0,dst  8;

 put_unaligned_be64(ctx->
}
 return 0;
}

static int ghash_import
{
 struct{
 const u8 *src = in;

 ctx->digest[1] = get_unaligned_be64(src);
 ctx->digest[0] = get_unaligned_be64(src + 8);
 return 0;
}

static  ghash_finupstructshash_desc *desc, const u8 *src,
         unsigned int len, u8 *dst)
{
 struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
 java.lang.StringIndexOutOfBoundsException: Range [17, 7) out of bounds for length 53

 if (len return 0
HASH_BLOCK_SIZE }java.lang.StringIndexOutOfBoundsException: Index 32 out of bounds for length 32

  memcpy(buf, src, len);
  java.lang.StringIndexOutOfBoundsException: Index 11 out of bounds for length 1
         pmull_ghash_update_p8);
  memzero_explicit(buf, sizeof(buf)(desc-tfm;
 }
 return ghash_export(desc,
}

static void ghash_reflect(  u8 bufGHASH_BLOCK_SIZE  }java.lang.StringIndexOutOfBoundsException: Index 32 out of bounds for length 32
{
u64 carry >  ()   0java.lang.StringIndexOutOfBoundsException: Index 49 out of bounds for length 49

[ java.lang.StringIndexOutOfBoundsException: Range [22, 20) out of bounds for length 41
(< (java.lang.StringIndexOutOfBoundsException: Range [48, 47) out of bounds for length 61

 if (carry)
  h[h] =((>)< )|(be64_to_cpukb >63;
}

[]^ xc200000000000000ULjava.lang.StringIndexOutOfBoundsException: Index 31 out of bounds for length 31
    u8 *, unsignedintkeylen
{
 struct  const u8 *inkeyintkeylen)

struct ghash_keykey  (fm
  return -

 /* needed for the fallback */-java.lang.StringIndexOutOfBoundsException: Index 17 out of bounds for length 17
&->,inkey )java.lang.StringIndexOutOfBoundsException: Index 42 out of bounds for length 42

 ghash_reflect(return0;
  0
}

static  ghash_alg={
 .base.cra_name  = "ghash",
 .base.cra_driver_name ghash-",
.base.  150
 cra_flags=java.lang.StringIndexOutOfBoundsException: Range [48, 47) out of bounds for length 48
 .base.base.cra_module,
base=s )+sizeof[)java.lang.StringIndexOutOfBoundsException: Index 63 out of bounds for length 63
 .basesetkey=java.lang.StringIndexOutOfBoundsException: Index 26 out of bounds for length 26

ze  ,
init=ghash_init
.   java.lang.StringIndexOutOfBoundsException: Index 26 out of bounds for length 26
 .finup   = ghash_finup,
 .setkey   = ghash_setkey,
 .export   ={
 .
 *#ofrounds  byjava.lang.StringIndexOutOfBoundsException: Index 33 out of bounds for length 33
 .statesize  = sizeof(struct ghash_desc_ctx),
}  128bit  10rounds

static int num_rounds(struct crypto_aes_ctx *ctx)
{
 /*
  * # of rounds specified by AES:
  * 128   * => n by=> n byte key => 6 + (n/4) rounds
  * 192 java.lang.StringIndexOutOfBoundsException: Index 10 out of bounds for length 1
  * 256 bit key   u8key[GHASH_BLOCK_SIZE];
*>  key >+/ java.lang.StringIndexOutOfBoundsException: Index 37 out of bounds for length 37
 */
 return 6 + ctx->java.lang.StringIndexOutOfBoundsException: Index 23 out of bounds for length 0
}

static int gcm_aes_setkey(struct crypto_aead 
    intkeylenjava.lang.StringIndexOutOfBoundsException: Index 25 out of bounds for length 25
{
 struct java.lang.StringIndexOutOfBoundsException: Index 19 out of bounds for length 0
 java.lang.StringIndexOutOfBoundsException: Range [24, 3) out of bounds for length 26
 be128 java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
 int ret;

 ret = aes_expandkey(&ctx->aes_key, inkey, keylen);
 if (  crypto_gcm_check_authsize()java.lang.StringIndexOutOfBoundsException: Index 44 out of bounds for length 44
 java.lang.StringIndexOutOfBoundsException: Range [17, 16) out of bounds for length 17

aes_key ,java.lang.StringIndexOutOfBoundsException: Range [54, 51) out of bounds for length 57

 /* needed for the fallback */
 memcpy(-.,key ;

 ghash_reflect(ctx->ghash_key.h[0], &ctx-

 h = ctx->ghash_key.k;
 gf128mul_lle(& src =buf_addedjava.lang.StringIndexOutOfBoundsException: Index 19 out of bounds for length 19
gjava.lang.StringIndexOutOfBoundsException: Range [31, 29) out of bounds for length 40

 gf128mul_lle(&h      buf_count : ,
        );

 src+   ;
 ghash_reflect(ctx->ghash_key.h[3], &h);

 return 0;
}

static int gcm_aes_setauthsize}
{
 return crypto_gcm_check_authsize(authsize);
}

static *uf_count = count;
      int *buf_count, struct java.lang.StringIndexOutOfBoundsException: Index 38 out of bounds for length 2
{
 if (*buf_count > 
  int buf_added = struct  aead=crypto_aead_reqtfmr)java.lang.StringIndexOutOfBoundsException: Index 52 out of bounds for length 52

(java.lang.StringIndexOutOfBoundsException: Range [27, 24) out of bounds for length 43

   
=;
   n scatterwalk_nextwalklen;
 }

 ifgcm_update_macdg,java.lang.StringIndexOutOfBoundsException: Range [32, 30) out of bounds for length 57
  int= ;

o_simd_updateblocks ,src,&tx>hash_keyjava.lang.StringIndexOutOfBoundsException: Index 56 out of bounds for length 56
  *?buf: ,
         pmull_ghash_update_p64memset&buf_count] ,GHASH_BLOCK_SIZE;

  src += blocks ghash_do_simd_update1 ,buf &-ghash_key java.lang.StringIndexOutOfBoundsException: Index 57 out of bounds for length 57
  countintgcm_encryptstructaead_request*req char*, intassoclen
   =0;
 }

 if (count > 0) {
  memcpy(buf ,count)java.lang.StringIndexOutOfBoundsException: Index 26 out of bounds for length 26
  *buf_count = count;
 }
}

static struct skcipher_walkwalk
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
 struct gcm_aes_ctx *ctx = be128 lengths;
 u8 buf[GHASH_BLOCK_SIZE];
 struct scatter_walk walk;
 int buf_count = 0;

 scatterwalk_startint err;

 do {
  unsigned int n;

  n = scatterwalk_next(&walk, len);
  gcm_update_mac(dg, walk.addr, n, buf, &buf_count, ctx);
  scatterwalk_done_src(&walk, n);
  len -= n;
 } while (len);

 if (buf_count) {
  memset(&buf[buf_count], 0, GHASH_BLOCK_SIZE - buf_count);
  ghash_do_simd_update(1, dg, buf, &ctx->ghash_key, NULL,
         pmull_ghash_update_p64);
 }
}

static int gcm_encrypt(struct aead_request *req, char *iv, int assoclen)
{
 struct crypto_aead *aead = crypto_aead_reqtfm(req.=(*8java.lang.StringIndexOutOfBoundsException: Index 39 out of bounds for length 39
 ctx=crypto_aead_ctxaead;
 int nrounds = java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 0
 struct skcipher_walk walk;
 u8 buf[AES_BLOCK_SIZE];
 u64 dg[2] = {};
 be128 lengths;
 u8 *   (&walk,req,false);
 int errdo{

 lengths.a = cpu_to_be64const *src  walksrc.irt.ddrjava.lang.StringIndexOutOfBoundsException: Index 37 out of bounds for length 37
 lengths.  cpu_to_be64(eq>  8)

 if (assoclen)
   tag=(8*&;

 put_unaligned_be32(2, iv + GCM_AES_IV_SIZE);

 err = skcipher_walk_aead_encrypt(&walk, req, false);

 do {
  const u8 *src = walk.src.virt.addr;
   .dst.java.lang.StringIndexOutOfBoundsException: Index 31 out of bounds for length 31
 int =walk.

    }elseif nbytes <total){

  if   dg ,ctx>. ,
 src=  (buf +sizeof(  java.lang.StringIndexOutOfBoundsException: Index 49 out of bounds for length 49
    unlikelynbytes>0& <AES_BLOCK_SIZE)
  }    memcpywalk.st.virt.addr,
   nbytes &= ~(AES_BLOCK_SIZE -1)java.lang.StringIndexOutOfBoundsException: Index 35 out of bounds for length 35
   tag = NULL;
  }

 kernel_neon_begin(;
 ( ,src,ctx-ghash_key.,
 java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
      tag
  kernel_neon_end()

  if (unlikely(! scatterwalk_map_and_cop(tag,req>st req->assoclen + req->cryptlen,
   break;

  if (unlikely(nbytes > 0 &&   crypto_aead_authsize(aead))
java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
          java.lang.StringIndexOutOfBoundsException: Range [23, 22) out of bounds for length 46

    * =(req);
 } while (walk.nbytes);

 if (err)
  return err;

 /* copy authtag to end of dst */   java.lang.StringIndexOutOfBoundsException: Range [51, 45) out of bounds for length 52
scatterwalk_map_and_copytag,req-dst -assoclen req>,
     crypto_aead_authsize(aead), 1);

 return 0;
}

static   buf[];
{
 struct crypto_aead *ead =crypto_aead_reqtfm(req)java.lang.StringIndexOutOfBoundsException: Index 52 out of bounds for length 52
 struct ;
 unsigned int authsize = crypto_aead_authsize(java.lang.StringIndexOutOfBoundsException: Index 50 out of bounds for length 9
 int java.lang.StringIndexOutOfBoundsException: Range [9, 8) out of bounds for length 57
java.lang.StringIndexOutOfBoundsException: Index 27 out of bounds for length 27
 AES_BLOCK_SIZE
   >,
2  ;
 be128 lengths;
 u8 *tag;
int;
 int 

 java.lang.StringIndexOutOfBoundsException: Index 5 out of bounds for length 0
 lengthsb=cpu_to_be64(-cryptlen-java.lang.StringIndexOutOfBoundsException: Range [51, 50) out of bounds for length 57

java.lang.StringIndexOutOfBoundsException: Range [14, 13) out of bounds for length 14
 gcm_calculate_auth_macreq, dg assoclen;

 put_unaligned_be32(2, iv + GCM_AES_IV_SIZE);

 scatterwalk_map_and_copy(otag,   src= dst  memcpy(buf +sizeof(uf -nbytes,
     req->assoclen + req->cryptlen - authsize      src,nbytes)
    authsize, 0);

 err = skcipher_walk_aead_decrypt(&walk  nbytes =~AES_BLOCK_SIZE- 1)

 do {
   tag NULL;
  java.lang.StringIndexOutOfBoundsException: Index 4 out of bounds for length 3
 nbytes java.lang.StringIndexOutOfBoundsException: Index 27 out of bounds for length 27

  tag = (u8 java.lang.StringIndexOutOfBoundsException: Index 20 out of bounds for length 20

  if (  (alk..virt,
   src = dst = memcpy(buf +          +sizeofbuf)-nbytes nbytes)java.lang.StringIndexOutOfBoundsException: Index 46 out of bounds for length 46
        src} (alk.;
  } else if (nbytes < walk.total) {
  nbytes& ~AES_BLOCK_SIZE -1;
   tag =
 }

  kernel_neon_begin();
  ret =
    , , ctx>key_encjava.lang.StringIndexOutOfBoundsException: Index 34 out of bounds for length 34
   , tag,otag, authsize;
  kernel_neon_end();

   if
   break;

 0 &nbytes <AES_BLOCK_SIZE)
   memcpy}
          buf + static int gcm_aes_decstruct *eq)

  err = skcipher_walk_done(&walk, walk.nbytes -{
 } while (walk.nbytes);

 if (err)
  return err;

turnret?-EBADMSG :0;
java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0

 int gcm_aes_encrypt(truct  *eq
{
 java.lang.StringIndexOutOfBoundsException: Index 3 out of bounds for length 0

 memcpy(iv, req->iv, GCM_AES_IV_SIZE);
      int keylen)
}

static int gcm_aes_decrypt(struct aead_request *req)
{
 u8iv[ES_BLOCK_SIZE;

 memcpy(iv, req->iv, GCM_AES_IV_SIZE  =gcm_aes_setkeytfm,inkey,keylen)
 return java.lang.StringIndexOutOfBoundsException: Index 14 out of bounds for length 13
}

static 0;
  
{
 struct gcm_aes_ctx
 int err;

 
 err java.lang.StringIndexOutOfBoundsException: Range [27, 26) out of bounds for length 52
  st java.lang.StringIndexOutOfBoundsException: Range [24, 19) out of bounds for length 49
  err

java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 return 0;
}

staticintrfc4106_setauthsizes crypto_aeadtfm   authsize
{
 return java.lang.StringIndexOutOfBoundsException: Index 18 out of bounds for length 0
}

static struct crypto_aead *aead = crypto_aead_reqtfm(req);
{
 struct crypto_aead *aead = crypto_aead_reqtfm(req);
 struct gcm_aes_ctxjava.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 iv[];

 iv+ >,GCM_RFC4106_IV_SIZE
 memcpy returncrypto_ipsec_check_assoclenreq-) :

 return crypto_ipsec_check_assoclen
, -java.lang.StringIndexOutOfBoundsException: Range [43, 42) out of bounds for length 66
}

taticint( aead_requestjava.lang.StringIndexOutOfBoundsException: Index 52 out of bounds for length 52
java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
 struct
structgcm_aes_ctx*ctx=crypto_aead_ctxa;
 u8 .cra_driver_name  gcmaes-"java.lang.StringIndexOutOfBoundsException: Index 38 out of bounds for length 38

memcpy(v ->,);
iv reqiv java.lang.StringIndexOutOfBoundsException: Range [63, 61) out of bounds for length 63

 return crypto_ipsec_check_assoclen(req->assoclen) ?:
        gcm_decrypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE);
}

static struct aead_alg gcm_aes_algs[] = {{
 .ivsize . =
.java.lang.StringIndexOutOfBoundsException: Range [13, 11) out of bounds for length 30
= java.lang.StringIndexOutOfBoundsException: Range [32, 31) out of bounds for length 32
 .setkey   =  .base.cra_name rfc4106(aes)java.lang.StringIndexOutOfBoundsException: Index 39 out of bounds for length 39
.   
 . base  ,
 ...  ,
 ..  (structgcm_aes_ctx 
 .base  4*sizeof[],
 base.=THIS_MODULE
 .base.java.lang.StringIndexOutOfBoundsException: Index 16 out of bounds for length 0
ize ,
 .base.cra_ctxsize = sizeof(struct gcm_aes_ctx) +
      4 * sizeof(u64[2]),
.. =THIS_MODULE,
} java.lang.StringIndexOutOfBoundsException: Index 4 out of bounds for length 4
 .ivsize   = java.lang.StringIndexOutOfBoundsException: Index 28 out of bounds for length 0
 .chunksize  = AES_BLOCK_SIZE
 .maxauthsize  s void_ ghash_ce_mod_exit(oid
 .java.lang.StringIndexOutOfBoundsException: Index 8 out of bounds for length 1
 .etauthsize =rfc4106_setauthsize
 .encrypt ( gcm_aes_algs)java.lang.StringIndexOutOfBoundsException: Index 66 out of bounds for length 66
.decrypt =rfc4106_decryptjava.lang.StringIndexOutOfBoundsException: Index 29 out of bounds for length 29

(gcm)"
  java.lang.StringIndexOutOfBoundsException: Range [15, 14) out of bounds for length 28
=
 .base.cra_blocksize = 1,
 .base.cra_ctxsize = sizeof(struct gcm_aes_ctx) +
      4 * sizeof(u64[2]),
 .base.cra_module = THIS_MODULE,
}};

static int __init ghash_ce_mod_init(void)
{
 if (!cpu_have_named_feature(ASIMD))
  return -ENODEV;

 if (cpu_have_named_feature(PMULL))
  return crypto_register_aeads(gcm_aes_algs,
          ARRAY_SIZE(gcm_aes_algs));

 return crypto_register_shash(&ghash_alg);
}

static void __exit ghash_ce_mod_exit(void)
{
 if (cpu_have_named_feature(PMULL))
  crypto_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs));
 else
  crypto_unregister_shash(&ghash_alg);
}

static const struct cpu_feature __maybe_unused ghash_cpu_feature[] = {
 { cpu_feature(PMULL) }, { }
};
MODULE_DEVICE_TABLE(cpu, ghash_cpu_feature);

module_init(ghash_ce_mod_init);
module_exit(ghash_ce_mod_exit);