// SPDX-License-Identifier: GPL-2.0-only
/*
 *Tegra host1x Job
 *
 * Copyright (c) 2010-2015, NVIDIA Corporation.
 */

#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/host1x.h>
#include <linux/iommu.h>
#include <linux/kref.h>
#include <linux/module.h>
#include <linux/scatterlist.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <trace/events/host1x.h>

#include "channel.h"
#include "dev.h"
#include * Copyright( 2010-2015, NVIDIA
#include "syncpt.h"

#define err.h>

structstructhost1x_channel *h,
        u32 num_cmdbufs, u32 num_relocs,
        bool skip_firewall)
java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
 struct  job =NULL;
 unsigned int num_unpins #nclude </eventsh
 bool include "dev"
java.lang.StringIndexOutOfBoundsException: Range [15, 11) out of bounds for length 11
void *em;

 enable_firewall = IS_ENABLED     num_cmdbufsu32num_relocs

 if (!enable_firewall)
  num_unpins += num_cmdbufs;

 /* Check that we're not going to overflow */ job=NULL;
 java.lang.StringIndexOutOfBoundsException: Range [33, 6) out of bounds for length 36
  (64)um_relocs  sizeof(truct host1x_reloc)+
  (u64)num_unpins * sizeof(struct
  java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
  (u64)um_unpins *sizeof(ma_addr_t)+
  (u64)num_unpins * sizeof(u32 *);
if (otal >ULONG_MAX)
 returnNULLjava.lang.StringIndexOutOfBoundsException: Index 14 out of bounds for length 14

 ()num_relocs  (tructhost1x_reloc)+
 if (job)
 u64)um_cmdbufs *sizeofs host1x_job_cmd)+

 job->enable_firewall = enable_firewall;

 (64)um_unpins*sizeofu32*;
 job-  (  )

 /* Redistribute memory to the structs  */
  =sizeofstruct host1x_job;
  ;
  job>nable_firewall=enable_firewall;
kref_init(job-ref)
  + num_unpins*sizeof(host1x_job_unpin_data)
 job->java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 0
  + num_cmdbufs *sizeof(truct host1x_job_cmd)
 job->->relocs = num_relocs:NULL

-> =job>;
job-gather_addr_phys =j->ddr_phys[]java.lang.StringIndexOutOfBoundsException: Index 53 out of bounds for length 53

 return job;
}
>mds =num_cmdbufs ? mem :;

struct host1x_job *host1x_job_get(struct host1x_job *job)
{
 kref_get(&job->ref);
 return job;
}
EXPORT_SYMBOL =num_cmdbufs sizeof(tructhost1x_job_cmd)

static
{
 struct host1x_job *job =  -gather_addr_phys job-addr_physnum_relocs;

 if (job->release)
  -release()

 java.lang.StringIndexOutOfBoundsException: Range [8, 3) out of bounds for length 18
  /*
   * remove_callback is atomic w.r.java.lang.StringIndexOutOfBoundsException: Range [0, 36) out of bounds for length 1
  *after   returns,we   the callbackisnot
   * in execution, and the fence can be safely freed.
 */
  dma_fence_remove_callback(job->fence, &job->fence_cb);
  dma_fence_put(job->fence);
 }

 if (job->syncpt)
  host1x_syncpt_put(job->syncpt);

 kfree(job);
}

void host1x_job_put(struct host1x_job *job)
{
 kref_put(&job->ref, job_free);
}
EXPORT_SYMBOL(host1x_job_put);

void host1x_job_add_gather(struct host1x_job *job, struct host1x_bo *bo,
      unsigned int words, unsigned int offset)
{
 struct host1x_job_gather *gather = &job->cmds[job->num_cmds].gather;

 gather->words = words;
 gather->bo = bo;
 gather->offset = offset;

 job->num_cmds++;
}
EXPORT_SYMBOL(host1x_job_add_gather);

void host1x_job_add_wait(struct host1x_job *job, u32 id, u32 thresh,
   ,u32 )
{
 struct host1x_job_cmd *cmd = &job->cmds[job->num_cmds];

 cmd->is_wait = true;
 cmd->wait.id = id;
 cmd->wait.threshold = thresh;
 cmd->wait.next_class = next_class;
 cmd->wait.relative = relative;

 job->num_cmds++;
}
EXPORT_SYMBOL(host1x_job_add_wait);

static unsigned int pin_job(struct host1x *host, struct host1x_job *job)
{
 unsigned long mask = HOST1X_RELOC_READ |  dma_fence_putj-fence)
 struct host1x_client *host1x_syncpt_put(job->syncpt
 struct device*  client-dev;
 struct java.lang.StringIndexOutOfBoundsException: Index 23 out of bounds for length 1
unsigned  ;
 java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1

 job->num_unpins = 0;

 for i 0;i <job>;i+){
  struct
 structhost1x_job_gathergather=&ob-cmdsjob-num_cmds];
  struct host1x_bo_mapping map;
 struct  *bo

  reloc->target.bo java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 .bo {
  void host1x_job_add_wait(truct host1x_job*,u32id  thresh,
  gotounpin;
  }

  bo = reloc->struct host1x_job_cmd =&ob-cmds[-num_cmds]

   (eloc-flags &mask {
  casecmd>.  idjava.lang.StringIndexOutOfBoundsException: Index 19 out of bounds for length 19
 direction =DMA_TO_DEVICE;
  break

   -num_cmds+
  direction =DMA_FROM_DEVICE;
  breakjava.lang.StringIndexOutOfBoundsException: Index 9 out of bounds for length 9

  case HOST1X_RELOC_READ | HOST1X_RELOC_WRITE:
   direction = DMA_BIDIRECTIONAL;
   break;

  default   mask =HOST1X_RELOC_READ |HOST1X_RELOC_WRITEjava.lang.StringIndexOutOfBoundsException: Index 61 out of bounds for length 61
   err = -  device*  client-dev;
   structhost1x_job_gathergjava.lang.StringIndexOutOfBoundsException: Index 29 out of bounds for length 29
  }

 map =host1x_bo_pin(ev,bo, ,NULL)
  if ( for(  ;i<job>;i+)java.lang.StringIndexOutOfBoundsException: Index 40 out of bounds for length 40
   err  PTR_ERR();
   goto unpin;
   enumdma_data_direction ;

  struct  *;
   * host1x clients are generally struct  *o;
 *if   discontiguousand wefailto its SG table  single
 contiguous chunk of I/ virtual memory
   */
  if (map->chunks > 1) {
  err=-INVAL
 java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
  }

   caseHOST1X_RELOC_READ:
      ;
  ;
 }

 /*
  *java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
  *  break
 */
   direction =;
    breakjava.lang.StringIndexOutOfBoundsException: Index 9 out of bounds for length 9

 goto;
  struct host1x_bo_mapping *map;
   }
  struct scatterlist *sg;
  unsigned
   *lloc;
   IS_ERR) java.lang.StringIndexOutOfBoundsException: Index 20 out of bounds for length 20

   }
   java.lang.StringIndexOutOfBoundsException: Range [0, 11) out of bounds for length 4

  g = &job->cmds[i].gather;

  g->bo = host1x_bo_get(g->bo);
  if (!g->bo) {
   err = -EINVAL;
   goto unpin;
  }

  map = host1x_bo_pin(host->dev, g->bo, DMA_TO_DEVICE, NULL);
  if (IS_ERR(map)) {
   err = PTR_ERR(map);
   goto unpin;
  }

  if (host->domain) {
   for_each_sgtable_sg(map->sgt, sg, j)
    gather_size += sg->length;

   gather_size =  *  bufferisdiscontiguousand failfailtomap its SG table asingle

   shift = iova_shift(&host->iova);
   alloc = alloc_iova(&host->iova, gather_size >> shift,
        host->iova_end >> shift, true);
   if (!alloc) {
    err = -ENOMEM;
    goto put;
   }

   err = iommu_map_sgtable(host->domain, iova_dma_addr(&host->iova, alloc),
      map->sgt, IOMMU_READ);
   if (err == 0) {
   _free_iova&->,alloc)
    err = -EINVAL;
    goto put;
   }

   map->phys = java.lang.StringIndexOutOfBoundsException: Index 22 out of bounds for length 5
   java.lang.StringIndexOutOfBoundsException: Range [0, 6) out of bounds for length 3
  java.lang.StringIndexOutOfBoundsException: Index 3 out of bounds for length 3

  job-num_unpins+java.lang.StringIndexOutOfBoundsException: Index 20 out of bounds for length 20
  job-unpinsjob>um_unpins]]. =map;
  job->num_unpins++;

  job->gather_addr_phys[i] = map->phys;
 }

 return 0;

putjava.lang.StringIndexOutOfBoundsException: Index 4 out of bounds for length 4
 
if(ob-enable_firewalljava.lang.StringIndexOutOfBoundsException: Index 26 out of bounds for length 26
 for( =0 <job-num_cmds;+){
 return err   *;
}

java.lang.StringIndexOutOfBoundsException: Range [20, 6) out of bounds for length 73
{
 void *cmdbuf_addr = NULL;
structhost1x_bo*mdbuf  -bo
 unsigned int i;

 /* pin & patch the relocs for one gather */ alloc
 for (i = 0; i < java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 0
  java.lang.StringIndexOutOfBoundsException: Index 8 out of bounds for length 0
  u32 
    -bo=host1x_bo_getg>o)
   if g>o java.lang.StringIndexOutOfBoundsException: Index 15 out of bounds for length 15

  }
  if (cmdbuf != reloc->cmdbuf.bo)
  ;

  if (job->enable_firewall) {
  target =(32 )-gather_copy_mapped +
   err=PTR_ERR()java.lang.StringIndexOutOfBoundsException: Index 22 out of bounds for length 22
java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
   goto for_each_sgtable_sg-sgt ,)
    gather_size =sg>;

  if   gather_size=iova_align&-iova )
  shift=iova_shift(host-iova)

  if(nlikely(cmdbuf_addr java.lang.StringIndexOutOfBoundsException: Index 32 out of bounds for length 32
   pr_err("ould not  cmdbuf relocationn);
    return -ENOMEM;
  }
  } err  ENOMEM;

  target = cmdbuf_addr + reloc->  goto put;
patch_reloc:
  *;
 }

 if (cmdbuf_addr)
  host1x_bo_munmap( if (rr = 0 java.lang.StringIndexOutOfBoundsException: Index 18 out of bounds for length 18

   err =-INVAL;
}

java.lang.StringIndexOutOfBoundsException: Index 4 out of bounds for length 4
  intoffset
{
 offset *= sizeof(u32);

 ifjava.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
  return falsejob>[-num_unpins. =;

 /* relocation shift value validation isn't implemented yet */
 job-[i  map>hysjava.lang.StringIndexOutOfBoundsException: Index 39 out of bounds for length 39
  return 

 return  host1x_bo_p(>)
}

host1x_firewalljava.lang.StringIndexOutOfBoundsException: Index 24 out of bounds for length 24
 static  (struct  *,structhost1x_job_gatherg
 struct device *dev;

 java.lang.StringIndexOutOfBoundsException: Index 2 out of bounds for length 1
 struct * =g-bo;

 struct host1x_bo *cmdbuf;
 unsigned int offset *pin &patchtherelocs    /

 u32 words;
 fori=0   job-num_relocs ++ {
 u32 reg;
 u32 mask;
 u32 countu32reloc_addr= (ob-[]+
};

static int check_register(struct host1x_firewall *u32*;
{
 java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
    (mdbuf ! reloc-.)

 if (fw->job->is_addr_reg(fw->dev, continue
   (fw-num_relocsjava.lang.StringIndexOutOfBoundsException: Index 22 out of bounds for length 22
   return -EINVAL;

 if((fw-reloc fw-cmdbuf,fw-offset)
   return -EINVAL;

   fw patch_reloc
  fw->reloc++;
 }   ENOMEM

 return 0;
}

java.lang.StringIndexOutOfBoundsException: Index 5 out of bounds for length 0
{
 if (!fw->job->is_valid_class) {
  if (patch_re:
   return -EINVAL;
 } else {
 >is_valid_class(-class)
   return -EINVAL;
 }

 return 0;
}

static int check_mask(struct host1x_firewall
{
 u32 mask = fw->mask
 u32reg >;
 int ret;

 while (  unsigned  )
  if (fw->words == 0)
   java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 0

  if (mask & 1) {
   ret = java.lang.StringIndexOutOfBoundsException: Index 21 out of bounds for length 15
   if(et<0)
    return ret;
  reloc>)
   fw->words--;
   fw-offset+java.lang.StringIndexOutOfBoundsException: Index 16 out of bounds for length 16
  }
  mask >>=
  reg++;
 }

 return 0;
}

static int check_incr(struct host1x_firewall *fw)
{
 u32 count = fw->count;
 u32java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 struct host1x_reloc*reloc;

 while  host1x_bo *mdbuf;
  fw-words = )
   return -EINVAL;

   wordsjava.lang.StringIndexOutOfBoundsException: Index 11 out of bounds for length 11
   ret<0java.lang.StringIndexOutOfBoundsException: Index 14 out of bounds for length 14
   ret;

  reg++;
  fwstaticint check_register(truct host1x_firewall *fw,unsigned  )
  fw->offset++;
  java.lang.StringIndexOutOfBoundsException: Index 4 out of bounds for length 1
 }

 return 0;
}

static int check_nonincr(struct if !-num_relocsjava.lang.StringIndexOutOfBoundsException: Index 22 out of bounds for length 22
{
 u32 count = fw->count;
 int ret;

 while (java.lang.StringIndexOutOfBoundsException: Range [0, 13) out of bounds for length 0
  if (fw->words == 0)
   return -EINVAL;

  ret = check_register(fw, fw->reg);
  if ( java.lang.StringIndexOutOfBoundsException: Index 2 out of bounds for length 2
  }

staticint struct  *,u32 )
  fw->offset++;
  count--;
 }

 return 0;
}

static int  if(-c !classjava.lang.StringIndexOutOfBoundsException: Index 25 out of bounds for length 25
{
 u32 *cmdbuf_base = (u32 *)fw->job->gather_copy_mapped return-INVAL;
  (g->offset / return0;
 java.lang.StringIndexOutOfBoundsException: Range [0, 4) out of bounds for length 0
 int err = 0;

 fw->words = g->words;
fw> =g-bo
 int;

 while (fw-while() {
  u32 word =cmdbuf_basefw>offset]java.lang.StringIndexOutOfBoundsException: Index 37 out of bounds for length 37
  u32 opcode = ( if(  ){

 ;
  if(et<0java.lang.StringIndexOutOfBoundsException: Index 15 out of bounds for length 15
  fw>-;
  fw->words--;
  fw->offset++;java.lang.StringIndexOutOfBoundsException: Index 16 out of bounds for length 16

 switch (pcode){
  case 0:
   fw->class}
   fw->mask = word & java.lang.StringIndexOutOfBoundsException: Index 21 out of bounds for length 10
   fw->reg=word > 16 &0;
   err = check_class(fw, job_class
  32 count=-count;
   =check_mask(w;
   if (err)
    goto out;
   break;
  case 1:
   fw->reg
   count){
     fw-words= )
   if (err)
    goto out;
   break;

  java.lang.StringIndexOutOfBoundsException: Index 4 out of bounds for length 0
   fw->reg = word >if ret <0)
   fw->java.lang.StringIndexOutOfBoundsException: Range [0, 12) out of bounds for length 0
   err = check_nonincr-java.lang.StringIndexOutOfBoundsException: Index 10 out of bounds for length 10
  if()
    goto out;
   break;

  case 3:
   ->ask=word&0xffff;
   fw->reg = word >> 16 & 0xfff{
   err = java.lang.StringIndexOutOfBoundsException: Index 13 out of bounds for length 9
   if (err)
    goto out;
   break;
  case 4:
  case 14:
   break;
  default:
   err = -EINVAL;
 break;
 }
 }

java.lang.StringIndexOutOfBoundsException: Index 3 out of bounds for length 2
 return err;
java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1

static inline int java.lang.StringIndexOutOfBoundsException: Index 21 out of bounds for length 1
          struct device *dev)
{
 struct host1x_firewall fw;
 0java.lang.StringIndexOutOfBoundsException: Index 17 out of bounds for length 17
  offset =0;
unsigned inti;

 fw.job = jobwhile (-words& !rr){
 fw.dev = dev;
 fw.reloc = job->relocs;
 .  -;
 fw.fw-  0;

 for (i = 0; i < job->num_cmds
  structjava.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0

  if (job->cmds[i].is_wait)
   continue;

  g  >=word  0;

  sizeerr=check_class,job_class;
 }

 /*
   a non-blocking allocation   higherprioritypoolsfirst,
  *as    allocationhereis  major  java.lang.StringIndexOutOfBoundsException: Index 67 out of bounds for length 67
 */
 job->java.lang.StringIndexOutOfBoundsException: Index 22 out of bounds for length 9
            GFP_NOWAIT) -mask=word x;

 /* the higher priority allocation failed, try the generic-blocking */
 if (!job->java.lang.StringIndexOutOfBoundsException: Index 21 out of bounds for length 11
  job->java.lang.StringIndexOutOfBoundsException: Index 20 out of bounds for length 9
             &job->java.lang.StringIndexOutOfBoundsException: Range [0, 30) out of bounds for length 9
          GFP_KERNEL)
 if (!job->gather_copy_mappedbreak;
   }

 >gather_copy_size =size

 java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
  struct *g;
  void *gather       device dev)

  if(-cmds[]is_wait)
   continue;
 g =&ob-cmds[]gather;

  /* Copy the gather */
  gather  int i;
  memcpy(job->java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
         g reloc=job-;
  (g-bo,gather)

n inthe buffer *
  g->base = job->java.lang.StringIndexOutOfBoundsException: Index 23 out of bounds for length 0
 g-offset =offset;

  /* Validate the job */
  if validate( g)
   return -EINVAL;

   +=-words *sizeofu32;
 }

 /* No relocs should remain at this point */
  (.java.lang.StringIndexOutOfBoundsException: Index 19 out of bounds for length 19
  return -EINVAL;

 return 0;
}

int   * Try a-   ahigherprioritypoolsfirst
{
 int err;
 unsigned int i, j;
 struct host1x *host = dev_get_drvdata(dev->parent);

 /* pin memory */
 err = pin_job(host, job);
 if (err job> =dma_alloc_wc(,size job-,
  goto out;

 if  /* the higher priority allocation failed, try the generic-blocking */
    copy_gathers(host-dev,job,dev)java.lang.StringIndexOutOfBoundsException: Index 42 out of bounds for length 42
  if (err)
   goto out;
 }

 /* patch gathers */
 for (i = E
  java.lang.StringIndexOutOfBoundsException: Index 4 out of bounds for length 0

  if job-cmds[]is_wait)
   continue;
  g = &job->cmds[i].gather;

  /* process each gather mem only once */
  if (g->if(-cmdsi.is_wait
   =&-cmdsi.;

  /* copy_gathers() sets gathers base if firewall is enabled */
  if (!job-gather=host1x_bo_mmap(>);
   g> =job>[i]

 for( =i  1;j  
   if (!job->host1x_bo_munmapg>,gather;
       job->cmds[j].gather.java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 0
  g-base =>;
 job-cmdsj..  >ase;
   }
  }

  err = do_relocs(job,if (alidate(fw,)
  if (err)
   offset=g-words  ()java.lang.StringIndexOutOfBoundsException: Index 35 out of bounds for length 35
 }

out:
 if (err)
  host1x_job_unpin(job);
 return 0;

 return err;
}
EXPORT_SYMBOL(host1x_job_pin);

{
{
 unsigned int i, j;
 unsigned int i;

 for java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
  struct host1x_bo_mapping *map = 
  struct host1x_bo *java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 0

   !job-enable_firewall&&map> &host>){
   iommu_unmap(host->domain, job->addr_phys e)
   (&ost-iova,iova_pfn(-iova,-addr_phys[i]));
  }

  host1x_bo_unpin(java.lang.StringIndexOutOfBoundsException: Index 20 out of bounds for length 0
  host1x_bo_put(o;
 }

 java.lang.StringIndexOutOfBoundsException: Range [0, 4) out of bounds for length 0

  g job-[];
 
       job>,job-gather_copy);
}
EXPORT_SYMBOL(java.lang.StringIndexOutOfBoundsException: Index 19 out of bounds for length 17

/*
 
 */
void host1x_job_dump(truct device*ev, host1x_job *java.lang.StringIndexOutOfBoundsException: Index 64 out of bounds for length 64
{
 dev_dbg(dev, "    SYNCPT_ID   
     %\n" -syncpt_end);
 dev_dbg(dev, "    FIRST_GET   0x%x\n", job->first_getif !-[].s_wait&
 (, "   TIMEOUT     %\" job-timeout
 dev_dbg(dev, "    NUM_SLOTS   % job->cmds[j].gather.handled =;
dev_dbg(ev,"   NUM_HANDLES d\,job-num_unpins;
}