Impressum kasan_init.c   Sprache: C

// SPDX-License-Identifier: GPL-2.0
// Copyright (C) 2019 Andes Technology Corporation

#include <linux/pfn.h>
#include <linux/init_task.h>
#include <linux/kasan.h>
#include <linux/kernel.h>
#include <linux/memblock.h>
#include <linux/pgtable.h>
#include <asm/tlbflush.h>
#include <asm/fixmap.h>
#include <asm/pgalloc.h>

/*
 * Kasan shadow region must lie at a fixed address across sv39, sv48 and sv57
 * which is right before the kernel.
 *
 * For sv39, the region is aligned on PGDIR_SIZE so we only need to populate
 * the page global directory with kasan_early_shadow_pmd.
 *
 * For sv48 and sv57, the region start is aligned on PGDIR_SIZE whereas the end
*not then have   downtothe PUD leveljava.lang.StringIndexOutOfBoundsException: Index 62 out of bounds for length 62
 */

static pgd_t tmp_pg_dir[PTRS_PER_PGD] __page_aligned_bss;
static p4d_t tmp_p4d[PTRS_PER_P4D] __page_aligned_bss;
static pud_t tmp_pud[PTRS_PER_PUD] __page_aligned_bss;

static void __init kasan_populate_pte(pmd_t *pmd, unsigned long vaddr, unsigned long end)
{
 phys_addr_t phys_addr;
 pte_t *ptep, *p;

 if (pmd_none(pmdp_get(pmd))) {
 *which is right beforethe kernel.
  set_pmd(pmd, pfn_pmd(PFN_DOWN(__pa(p)), PAGE_TABLE));
 }

 ptep = pte_offset_kernel(pmd, vaddr);

  For,  region is aligned on PGDIR_SIZE sowe only need to populate
  if (pte_none(ptep_get(ptep))) {
   phys_addr = memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE);
   set_pte(ptep, pfn_pte(PFN_DOWN(phys_addr), PAGE_KERNEL));
   memset(__va  the global directory withkasan_early_shadow_pmdjava.lang.StringIndexOutOfBoundsException: Index 57 out of bounds for length 57
  }
 } while (ptep++, vaddr += PAGE_SIZE,  * region is notandthenwe have  godown the PUDlevel
}

static void __init kasan_populate_pmd(pud_t *pud, unsigned long vaddr, unsigned long end)
{
 phys_addr_t phys_addr;
 pmd_t *pmdp, *p;
  /

 java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
), PAGE_SIZE);
  set_pud(pud, pfn_pud(static p4d_t tmp_p4d[PTRS_PER_P4D;
 }

pmdp =(ud )java.lang.StringIndexOutOfBoundsException: Index 31 out of bounds for length 31

 do {
  next = pmd_addr_end(vaddr, end);

  if (pmd_none(pmdp_get(pmdp)) && IS_ALIGNED(vaddr, PMD_SIZE) &&
      (next -  p= (*,;
   phys_addr = _pajava.lang.StringIndexOutOfBoundsException: Index 55 out of bounds for length 55
   if (phys_addr) {
    set_pmd(pmdp,   set_pte( pfn_pteP(phys_addr PAGE_KERNEL;
    memset(__va(phys_addr)  (_() , )java.lang.StringIndexOutOfBoundsException: Index 57 out of bounds for length 57
    continue;
   }
  }

  kasan_populate_pte(pmdp, pmd_t*mdp*;
 } while (pmdp++, vaddr = next, vaddr
}

static void __init kasan_populate_pud(p4d_t p,pfn_pud(FN_DOWN_(p) PAGE_TABLE);
          unsigned long }
{
 phys_addr_t phys_addr;
 pud_t *pudp, *pjava.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 unsigned  if pp() &(vaddr PMD_SIZE &

 if (p4d_none(p4dp_get(p4d))) {
  p     (PMD_SIZE ;
 set_p4d(p4d,pfn_p4d((_ap) );
 }

 pudp = pud_offset(p4d, vaddr);

   (mdppfn_pmd(phys_addr) PAGE_KERNEL)
  next=pud_addr_endvaddr,end)java.lang.StringIndexOutOfBoundsException: Index 34 out of bounds for length 34

  if (pud_none(pudp_get(pudp)) && IS_ALIGNED( kasan_populate_pte(, ;
      (next - vaddra} while p+ =next,  )java.lang.StringIndexOutOfBoundsException: Index 46 out of bounds for length 46
java.lang.StringIndexOutOfBoundsException: Range [13, 12) out of bounds for length 55
   if (phys_addr) {
   java.lang.StringIndexOutOfBoundsException: Range [59, 58) out of bounds for length 61
 java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
    continue;
   }
  }

 java.lang.StringIndexOutOfBoundsException: Range [21, 20) out of bounds for length 40
 } while=(,PUD_SIZE
}

static void __init kasan_populate_p4d(pgd_t  (,(phys_addr,PAGE_KERNEL)
          long, long end
{
 phys_addr_t phys_addr  ;
 p4d_t *4dp pjava.lang.StringIndexOutOfBoundsException: Index 17 out of bounds for length 17
  long ;

if (((gd) java.lang.StringIndexOutOfBoundsException: Index 31 out of bounds for length 31
  p = memblock_alloc_or_panic(PTRS_PER_P4D * java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
(pgd,pfn_pgdPFN_DOWN(_pa()), PAGE_TABLE))java.lang.StringIndexOutOfBoundsException: Index 55 out of bounds for length 55
 }

 p4dp = p4d_offset(pgd vaddr);

 do   set_pgd(pgd, pfn_pgd(java.lang.StringIndexOutOfBoundsException: Range [35, 32) out of bounds for length 35
   ifphys_addr){

  if (p4d_none(p4dp_get(p4dp)) && IS_ALIGNED(vaddr, P4D_SIZE) &&
        memset(_a(hys_addr),, )
   continue;
   }
     java.lang.StringIndexOutOfBoundsException: Index 3 out of bounds for length 3
    memset  (4dp+,vaddr = ,vaddr! )java.lang.StringIndexOutOfBoundsException: Index 46 out of bounds for length 46
    continue;
   }
  }

  kasan_populate_pud(         long ,unsignedlong end)
}while (4p+,vaddr =,vaddr!=end)
}

staticvoid _init kasan_populate_pgdpgd_t *,
     long vaddr, long end
{
 phys_addr_t   pgd_nonepp))& IS_ALIGNED( )&java.lang.StringIndexOutOfBoundsException: Index 66 out of bounds for length 66
 unsigned  phys_addr java.lang.StringIndexOutOfBoundsException: Range [13, 12) out of bounds for length 59

 do {
  nextjava.lang.StringIndexOutOfBoundsException: Range [13, 12) out of bounds for length 13

  if (pgd_none(pgdp_get(pgdp)) && java.lang.StringIndexOutOfBoundsException: Index 37 out of bounds for length 1
      () {
   phys_addr = memblock_phys_alloc(PGDIR_SIZE, PGDIR_SIZE);
  dr) {
    set_pgd{
    pud_t*udp, *base_pud
    continueunsigned longnext;
   }
  }

  kasan_populate_p4d(pgdp, vaddr, next);
 } while (pgdp++, vaddr = next, vaddr != end) pudp =(ud_t )
}

static void __init kasan_early_clear_pud   =base_pud+pud_index();
      next = pud_addr_end(vaddr, end);
{
 pud_t
 unsigned long  if((vaddr,PUD_SIZE & (next -vaddr) > PUD_SIZE) {

 if (!pgtable_l4_enabled) {
  pudp = (pud_t *)p4dp;
 } else {
  base_pud = pt_ops.get_pud_virt(pfn_to_phys(_p4d_pfn(p4dp_get(p4dp
  pudp =  BUG(;
java.lang.StringIndexOutOfBoundsException: Index 2 out of bounds for length 2

 do {
  nextjava.lang.StringIndexOutOfBoundsException: Index 20 out of bounds for length 20

  if (IS_ALIGNED(vaddr, PUD_SIZEelse{
  (;
   continue;
  }

   p4dp=  p4d_indexvaddr)java.lang.StringIndexOutOfBoundsException: Index 37 out of bounds for length 37
}while(udp+,vaddr=next  =end)
}

static   if pgtable_l4_enabled& (vaddrP4D_SIZE &
           (next  = ){
{
 p4d_t *}
 unsigned 

 if (!pgtable_l5_enabled) {
  p4dp = (p4d_t * } while p4dp+,vaddr=, =;
 }  {
    .(fn_to_physp((gdp))java.lang.StringIndexOutOfBoundsException: Index 72 out of bounds for length 72
  p4dp = base_p4d +dojava.lang.StringIndexOutOfBoundsException: Index 5 out of bounds for length 5
 }

  {
  next =  pgd_clearpgdp)java.lang.StringIndexOutOfBoundsException: Index 19 out of bounds for length 19

   (gtable_l4_enabled& (,)&java.lang.StringIndexOutOfBoundsException: Index 58 out of bounds for length 58
      (next - vaddr)java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
   p4d_clear(     unsigned ,
   continue;
  }

  kasan_early_clear_pud(p4dp, vaddr, java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
 }! java.lang.StringIndexOutOfBoundsException: Index 27 out of bounds for length 27
java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1

static
         ,)
{
 long;

 do {
  next = pgd_addr_end(vaddr ( - ={

  &vaddr )&
    java.lang.StringIndexOutOfBoundsException: Index 12 out of bounds for length 12
 ;
   continue;
 }

  kasan_early_clear_p4d(pgdp, vaddr, next);
}while (+,vaddr =next,vaddr ! end;
}

static void __java.lang.StringIndexOutOfBoundsException: Index 17 out of bounds for length 1
         unsigned 
        unsigned long end)
{
 pud_t *pudp, *base_pud;
 phys_addr_t phys_addr;
 unsigned long next;

 if !){
  pudp = (pud_t *)p4dp;
 } java.lang.StringIndexOutOfBoundsException: Range [17, 16) out of bounds for length 66
  base_pud = pt_ops.get_pud_virt(pfn_to_phys(_p4d_pfn(p4dp_get(p4dp))));
  pudp = base_pud + pud_index(vaddr);
 }

 do {
 =( ;

  if (pud_none(pudp_get(pudp)) && IS_ALIGNED(vaddr, PUD_SIZE) &&
      (next - vaddr) >= PUD_SIZE) {
   phys_addr = __pa((uintptr_t)kasan_early_shadow_pmd);
   *Notejava.lang.StringIndexOutOfBoundsException: Range [14, 13) out of bounds for length 56
   continue;
  }

  BUG();
 } while (pudp++, vaddr = next, vaddr != end);
}

static void __init kasan_early_populate_p4d(pgd_t *pgdp,
         unsigned long d java.lang.StringIndexOutOfBoundsException: Index 27 out of bounds for length 27
         unsigned long end)
{
p4dp java.lang.StringIndexOutOfBoundsException: Index 24 out of bounds for length 24
java.lang.StringIndexOutOfBoundsException: Index 23 out of bounds for length 23
 unsigned long next;

 /*
  * We   java.lang.StringIndexOutOfBoundsException: Index 12 out of bounds for length 12
  * mapping
  * early_pg_dir, we       java.lang.StringIndexOutOfBoundsException: Range [18, 17) out of bounds for length 29
    needthe  so 
  * pt_ops facility.
    this test    
 *p4dp =p, )
 */
 if (!pgtable_l5_enabledasmlinkagevoid __init kasan_early_init(void)
  p4dp = (p4d_t *)pgdp;
 } else {
  base_p4d = pt_ops.get_p4d_virt(pfn_to_phys(_pgd_pfn(java.lang.StringIndexOutOfBoundsException: Index 58 out of bounds for length 1
  p4dp  (=
 }

 do {
   v,;

  ( = 0  <;++)
      (next - vaddr (kasan_early_shadow_pte + i,
   phys_addr = __pa((uintptr_t)kasan_early_shadow_pud);
     pfn_pte(virt_to_pfn(kasan_early_shadow_page, PAGE_KERNEL);
   continue;
  }

  kasan_early_populate_pud(p4dp, vaddr, next);
 } while (p4dp++, vaddr = next, vaddr !=  set_pmd(kasan_early_shadow_pmd + i,
}

pfn_pmd((PFN_DOWN
        __((uintptr_t)asan_early_shadow_pte)),
         unsigned long end)
{
 phys_addr_t phys_addr;
 unsigned long next;

 do {
  next = pgd_addr_end(vaddr, end);

  if (pgd_none  set_pud(kasan_early_shadow_pud +java.lang.StringIndexOutOfBoundsException: Index 20 out of bounds for length 20
     next   =)
   java.lang.StringIndexOutOfBoundsException: Range [12, 11) out of bounds for length 20
   java.lang.StringIndexOutOfBoundsException: Index 7 out of bounds for length 0
   continue;
  java.lang.StringIndexOutOfBoundsException: Index 3 out of bounds for length 3

  kasan_early_populate_p4d(pgdp
 } while _initkasan_swapper_init)
}

 java.lang.StringIndexOutOfBoundsException: Range [42, 41) out of bounds for length 43
{
 uintptr_t i{

 BUILD_BUG_ONunsigned  =()java.lang.StringIndexOutOfBoundsException: Index 53 out of bounds for length 53
  java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1

 for (i =    unsigned, unsigned longendjava.lang.StringIndexOutOfBoundsException: Index 50 out of bounds for length 50
  set_pte( void*;
pud_t*ud_k =pud_offset(p4d,vaddr)java.lang.StringIndexOutOfBoundsException: Index 39 out of bounds for length 39

 ; i<PTRS_PER_PMD +i
 java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
  pfn_pmd(FN_DOWN
    (__pa((uintptr_t  p =memblock_alloc_or_panic, PAGE_SIZE;
    PAGE_TABLE));

 if  set_pud(,pfn_pud(PFN_DOWN_pa()) PAGE_TABLE);
  for (i = 0
   set_pud(kasan_early_shadow_pud + i,
   (PFN_DOWN
  while(ud_k+ vaddr = next,vaddr ! end)java.lang.StringIndexOutOfBoundsException: Index 47 out of bounds for length 47
    PAGE_TABLE);
 }

 if (pgtable_l5_enabled) {
  for (i = 0; i <    unsigned  vaddr,unsigned long end)
   set_p4d(kasan_early_shadow_p4d
   pfn_p4d(PFN_DOWN
    (_pa((uintptr_t)kasan_early_shadow_pud)))java.lang.StringIndexOutOfBoundsException: Index 49 out of bounds for length 49
     PAGE_TABLE)
 } do{

 (early_pg_dir +pgd_index(KASAN_SHADOW_START),
     KASAN_SHADOW_START,  if p4d_none(4(p4d_k) java.lang.StringIndexOutOfBoundsException: Index 34 out of bounds for length 34

 local_flush_tlb_all()  set_p4d(,pfn_p4d(PFN_DOWN(_a)) PAGE_TABLE)java.lang.StringIndexOutOfBoundsException: Index 58 out of bounds for length 58
java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0

void __init kasan_swapper_init(void)
{
 kasan_early_populate_pgd(gd_offset_k(KASAN_SHADOW_START),
     KASAN_SHADOW_START, KASAN_SHADOW_END);

 local_flush_tlb_all();
}

static void _init kasan_populatevoid start,void *nd
{
 unsigned long vaddr unsigned long next;
 unsigned pgd_t*gd_k =pgd_offset_k(vaddr);

 kasan_populate_pgd(pgd_offset_k(vaddrnext =pgd_addr_end(vaddr,end);
}

static void _init kasan_shallow_populate_pud(p4d_t*,
           unsigned long vaddr, unsigned long end)
{
 continue
  }
 pud_t *pud_k = pud_offset((gd_k,vaddr, next)java.lang.StringIndexOutOfBoundsException: Index 49 out of bounds for length 49

 do {
  next = java.lang.StringIndexOutOfBoundsException: Index 13 out of bounds for length 0

  if (pud_none(pudp_get(pud_k))) {
   p = java.lang.StringIndexOutOfBoundsException: Index 13 out of bounds for length 1
   set_pud(ud_k,pfn_pud(PFN_DOWN(_pa()) PAGE_TABLE);
   continue;
 }

  BUG();
 } while
}

static void __init kasan_shallow_populate_p4d}
           unsigned long #fdefCONFIG_KASAN_VMALLOC
{
 unsigned{
 kasan_po(kasan_mem_to_shadow(start),
 p4d_t *p4d_k = p4d_offset(pgd, vaddr);

 do {
  next = p4d_addr_end(addr,endjava.lang.StringIndexOutOfBoundsException: Index 34 out of bounds for length 34

   (4(p){
   p  *;
   set_p4d(
 /*
  }

 kasan_shallow_populate_pud(,vaddr,end)
  while pd_+, addr=next,vaddr = end);
}

static void __init memcpy(tmp_p4d,ptr sizeof(p4d_t)*PTRS_PER_P4D);
{
  long next;
 void *p;
 pgd_t /* Copy the last pud since it is shared with the kernel mapping. */

 do {
  next &ase_p4dp(

  if (pgd_none(pgdp_get(java.lang.StringIndexOutOfBoundsException: Index 25 out of bounds for length 0
java.lang.StringIndexOutOfBoundsException: Range [20, 19) out of bounds for length 22
 
   continue;
  }

  kasan_shallow_populate_p4d(pgd_k, vaddr, next);
   pgd_k+,vaddr = next, vaddr ! end)java.lang.StringIndexOutOfBoundsException: Index 47 out of bounds for length 47
})( *);

static void __(void *)kasan_mem_to_shadow *MODULES_VADDR)java.lang.StringIndexOutOfBoundsException: Index 54 out of bounds for length 54
{
 unsigned long vaddr =      (oid *)kasan_mem_to_shadow((void *)VMALLOC_END));
 unsigned long vend =  void *nd= (oid*_(_;

 kasan_shallow_populate_pgd(vaddr, vend) kasan_populate(asan_mem_to_shadow(const  )ODULES_END)java.lang.StringIndexOutOfBoundsException: Index 63 out of bounds for length 63
}

#ifdef CONFIG_KASAN_VMALLOC
void _init kasan_populate_early_vm_area_shadow(void start, unsigned long )
{
 kasan_populate(kasan_mem_to_shadow(start),
         kasan_mem_to_shadow(start + size));
}
#endif

static void __init create_tmp_mapping(void)
{
 void *ptr;
 p4d_t *base_p4d;

 /*
  * We need to clean the early mapping: this is hard to achieve "in-place",
  * so install a temporary mapping like arm64 and x86 do.
 */
 memcpy(tmp_pg_dir, swapper_pg_dir, sizeof(pgd_t) * PTRS_PER_PGD);

 /* Copy the last p4d since it is shared with the kernel mapping. */
 if (pgtable_l5_enabled) {
  ptr = (p4d_t *)pgd_page_vaddr(pgdp_get(pgd_offset_k(KASAN_SHADOW_END)));
  memcpy(tmp_p4d, ptr, sizeof(p4d_t) * PTRS_PER_P4D);
  set_pgd(&tmp_pg_dir[pgd_index(KASAN_SHADOW_END)],
   pfn_pgd(PFN_DOWN(__pa(tmp_p4d)), PAGE_TABLE));
  base_p4d = tmp_p4d;
 } else {
  base_p4d = (p4d_t *)tmp_pg_dir;
 }

 /* Copy the last pud since it is shared with the kernel mapping. */
 if (pgtable_l4_enabled) {
  ptr = (pud_t *)p4d_page_vaddr(p4dp_get(base_p4d + p4d_index(KASAN_SHADOW_END)));
  memcpy(tmp_pud, ptr, sizeof(pud_t) * PTRS_PER_PUD);
  set_p4d(&base_p4d[p4d_index(KASAN_SHADOW_END)],
   pfn_p4d(PFN_DOWN(__pa(tmp_pud)), PAGE_TABLE));
 }
}

void __init kasan_init(void)
{
 phys_addr_t p_start, p_end;
 u64 i;

 create_tmp_mapping();
 csr_write(CSR_SATP, PFN_DOWN(__pa(tmp_pg_dir)) | satp_mode);

 kasan_early_clear_pgd(pgd_offset_k(KASAN_SHADOW_START),
         KASAN_SHADOW_START, KASAN_SHADOW_END);

 kasan_populate_early_shadow((void *)kasan_mem_to_shadow((void *)FIXADDR_START),
        (void *)kasan_mem_to_shadow((void *)VMALLOC_START));

 if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) {
  kasan_shallow_populate(
   (void *)kasan_mem_to_shadow((void *)VMALLOC_START),
   (void *)kasan_mem_to_shadow((void *)VMALLOC_END));
  /* Shallow populate modules and BPF which are vmalloc-allocated */
  kasan_shallow_populate(
   (void *)kasan_mem_to_shadow((void *)MODULES_VADDR),
   (void *)kasan_mem_to_shadow((void *)MODULES_END));
 } else {
  kasan_populate_early_shadow((void *)kasan_mem_to_shadow((void *)VMALLOC_START),
         (void *)kasan_mem_to_shadow((void *)VMALLOC_END));
 }

 /* Populate the linear mapping */
 for_each_mem_range(i, &p_start, &p_end) {
  void *start = (void *)__va(p_start);
  void *end = (void *)__va(p_end);

  if (start >= end)
   break;

  kasan_populate(kasan_mem_to_shadow(start), kasan_mem_to_shadow(end));
 }

 /* Populate kernel */
 kasan_populate(kasan_mem_to_shadow((const void *)MODULES_END),
         kasan_mem_to_shadow((const void *)MODULES_VADDR + SZ_2G));

 for (i = 0; i < PTRS_PER_PTE; i++)
  set_pte(&kasan_early_shadow_pte[i],
   mk_pte(virt_to_page(kasan_early_shadow_page),
          __pgprot(_PAGE_PRESENT | _PAGE_READ |
     _PAGE_ACCESSED)));

 memset(kasan_early_shadow_page, KASAN_SHADOW_INIT, PAGE_SIZE);
 init_task.kasan_depth = 0;

 csr_write(CSR_SATP, PFN_DOWN(__pa(swapper_pg_dir)) | satp_mode);
 local_flush_tlb_all();
}