/** * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address * it is mapped to. * @dev: drm device * @data: ioctl data blob * @file: drm file * * While the mapping holds a reference on the contents of the object, it doesn't * imply a ref on the object itself. * * IMPORTANT: * * DRM driver writers who look a this function as an example for how to do GEM * mmap support, please don't implement mmap support like here. The modern way * to implement DRM mmap support is with an mmap offset ioctl (like * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly. * That way debug tooling like valgrind will understand what's going on, hiding * the mmap call in a driver private ioctl will break that. The i915 driver only * does cpu mmaps this way because we didn't know better.
*/ int
i915_gem_mmap_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
{ struct drm_i915_private *i915 = to_i915(dev); struct drm_i915_gem_mmap *args = data; struct drm_i915_gem_object *obj; unsignedlong addr;
/* * mmap ioctl is disallowed for all discrete platforms, * and for all platforms with GRAPHICS_VER > 12.
*/ if (IS_DGFX(i915) || GRAPHICS_VER_FULL(i915) > IP_VER(12, 0)) return -EOPNOTSUPP;
if (args->flags & ~(I915_MMAP_WC)) return -EINVAL;
if (args->flags & I915_MMAP_WC && !pat_enabled()) return -ENODEV;
obj = i915_gem_object_lookup(file, args->handle); if (!obj) return -ENOENT;
/* prime objects have no backing filp to GEM mmap * pages from.
*/ if (!obj->base.filp) {
addr = -ENXIO; goto err;
}
/** * i915_gem_mmap_gtt_version - report the current feature set for GTT mmaps * * A history of the GTT mmap interface: * * 0 - Everything had to fit into the GTT. Both parties of a memcpy had to * aligned and suitable for fencing, and still fit into the available * mappable space left by the pinned display objects. A classic problem * we called the page-fault-of-doom where we would ping-pong between * two objects that could not fit inside the GTT and so the memcpy * would page one object in at the expense of the other between every * single byte. * * 1 - Objects can be any size, and have any compatible fencing (X Y, or none * as set via i915_gem_set_tiling() [DRM_I915_GEM_SET_TILING]). If the * object is too large for the available space (or simply too large * for the mappable aperture!), a view is created instead and faulted * into userspace. (This view is aligned and sized appropriately for * fenced access.) * * 2 - Recognise WC as a separate cache domain so that we can flush the * delayed writes via GTT before performing direct access via WC. * * 3 - Remove implicit set-domain(GTT) and synchronisation on initial * pagefault; swapin remains transparent. * * 4 - Support multiple fault handlers per object depending on object's * backing storage (a.k.a. MMAP_OFFSET). * * 5 - Support multiple partial mmaps(mmap part of BO + unmap a offset, multiple * times with different size and offset). * * Restrictions: * * * snoopable objects cannot be accessed via the GTT. It can cause machine * hangs on some architectures, corruption on others. An attempt to service * a GTT page fault from a snoopable object will generate a SIGBUS. * * * the object must be able to fit into RAM (physical memory, though no * limited to the mappable aperture). * * * Caveats: * * * a new GTT page fault will synchronize rendering from the GPU and flush * all data to system memory. Subsequent access will not be synchronized. * * * all mappings are revoked on runtime device suspend. * * * there are only 8, 16 or 32 fence registers to share between all users * (older machines require fence register for display and blitter access * as well). Contention of the fence registers will cause the previous users * to be unmapped and any new access will generate new page faults. * * * running out of memory while servicing a fault may generate a SIGBUS, * rather than the expected SIGSEGV.
*/ int i915_gem_mmap_gtt_version(void)
{ return 5;
}
/* If the partial covers the entire object, just create a normal VMA. */ if (chunk >= obj->base.size >> PAGE_SHIFT)
view.type = I915_GTT_VIEW_NORMAL;
return view;
}
static vm_fault_t i915_error_to_vmf_fault(int err)
{ switch (err) { default:
WARN_ONCE(err, "unhandled error in %s: %i\n", __func__, err);
fallthrough; case -EIO: /* shmemfs failure from swap device */ case -EFAULT: /* purged object */ case -ENODEV: /* bad object, how did you get here! */ case -ENXIO: /* unable to access backing store (on device) */ return VM_FAULT_SIGBUS;
case -ENOMEM: /* our allocation failure */ return VM_FAULT_OOM;
case 0: case -EAGAIN: case -ENOSPC: /* transient failure to evict? */ case -ENOBUFS: /* temporarily out of fences? */ case -ERESTARTSYS: case -EINTR: case -EBUSY: /* * EBUSY is ok: this just means that another thread * already did the job.
*/ return VM_FAULT_NOPAGE;
}
}
/* Sanity check that we allow writing into this object */ if (unlikely(i915_gem_object_is_readonly(obj) &&
area->vm_flags & VM_WRITE)) return VM_FAULT_SIGBUS;
if (i915_gem_object_lock_interruptible(obj, NULL)) return VM_FAULT_NOPAGE;
err = i915_gem_object_pin_pages(obj); if (err) goto out;
/* * Let's move into the ">> PAGE_SHIFT" * domain to be sure not to lose bits
*/
vm_start = area->vm_start >> PAGE_SHIFT;
vm_end = area->vm_end >> PAGE_SHIFT;
vma_size = vma->size >> PAGE_SHIFT;
/* * Calculate the memory boundaries by considering the offset * provided by the user during memory mapping and the offset * provided for the partial mapping.
*/
start = vm_start;
start -= obj_offset;
start += vma->gtt_view.partial.offset;
end = start + vma_size;
start = max_t(long, start, vm_start);
end = min_t(long, end, vm_end);
/* Let's move back into the "<< PAGE_SHIFT" domain */
*start_vaddr = (unsignedlong)start << PAGE_SHIFT;
*end_vaddr = (unsignedlong)end << PAGE_SHIFT;
i915_gem_ww_ctx_init(&ww, true);
retry:
ret = i915_gem_object_lock(obj, &ww); if (ret) goto err_rpm;
/* Sanity check that we allow writing into this object */ if (i915_gem_object_is_readonly(obj) && write) {
ret = -EFAULT; goto err_rpm;
}
ret = i915_gem_object_pin_pages(obj); if (ret) goto err_rpm;
ret = intel_gt_reset_lock_interruptible(ggtt->vm.gt, &srcu); if (ret) goto err_pages;
/* Now pin it into the GTT as needed */
vma = i915_gem_object_ggtt_pin_ww(obj, &ww, NULL, 0, 0,
PIN_MAPPABLE |
PIN_NONBLOCK /* NOWARN */ |
PIN_NOEVICT); if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK)) { /* Use a partial view if it is bigger than available space */ struct i915_gtt_view view =
compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES); unsignedint flags;
flags = PIN_MAPPABLE | PIN_NOSEARCH; if (view.type == I915_GTT_VIEW_NORMAL)
flags |= PIN_NONBLOCK; /* avoid warnings for pinned */
/* * Userspace is now writing through an untracked VMA, abandon * all hope that the hardware is able to track future writes.
*/
/* * The entire mappable GGTT is pinned? Unexpected! * Try to evict the object we locked too, as normally we skip it * due to lack of short term pinning inside execbuf.
*/ if (vma == ERR_PTR(-ENOSPC)) {
ret = mutex_lock_interruptible(&ggtt->vm.mutex); if (!ret) {
ret = i915_gem_evict_vm(&ggtt->vm, &ww, NULL);
mutex_unlock(&ggtt->vm.mutex);
} if (ret) goto err_reset;
vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags);
}
} if (IS_ERR(vma)) {
ret = PTR_ERR(vma); goto err_reset;
}
/* Access to snoopable pages through the GTT is incoherent. */ /* * For objects created by userspace through GEM_CREATE with pat_index * set by set_pat extension, coherency is managed by userspace, make * sure we don't fail handling the vm fault by calling * i915_gem_object_has_cache_level() which always return true for such * objects. Otherwise this helper function would fall back to checking * whether the object is un-cached.
*/ if (!(i915_gem_object_has_cache_level(obj, I915_CACHE_NONE) ||
HAS_LLC(i915))) {
ret = -EFAULT; goto err_unpin;
}
ret = i915_vma_pin_fence(vma); if (ret) goto err_unpin;
/* * Dump all the necessary parameters in this function to perform the * arithmetic calculation for the virtual address start and end and * the PFN (Page Frame Number).
*/
set_address_limits(area, vma, obj_offset, ggtt->gmadr.start,
&start, &end, &pfn);
/* Finally, remap it using the new GTT offset */
ret = remap_io_mapping(area, start, pfn, end - start, &ggtt->iomap); if (ret) goto err_fence;
assert_rpm_wakelock_held(rpm);
/* Mark as being mmapped into userspace for later revocation */
mutex_lock(&to_gt(i915)->ggtt->vm.mutex); if (!i915_vma_set_userfault(vma) && !obj->userfault_count++)
list_add(&obj->userfault_link, &to_gt(i915)->ggtt->userfault_list);
mutex_unlock(&to_gt(i915)->ggtt->vm.mutex);
/* Track the mmo associated with the fenced vma */
vma->mmo = mmo;
if (CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)
intel_wakeref_auto(&i915->runtime_pm.userfault_wakeref,
msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND));
if (write) {
GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
i915_vma_set_ggtt_write(vma);
obj->mm.dirty = true;
}
err_fence:
i915_vma_unpin_fence(vma);
err_unpin:
__i915_vma_unpin(vma);
err_reset:
intel_gt_reset_unlock(ggtt->vm.gt, srcu);
err_pages:
i915_gem_object_unpin_pages(obj);
err_rpm: if (ret == -EDEADLK) {
ret = i915_gem_ww_ctx_backoff(&ww); if (!ret) goto retry;
}
i915_gem_ww_ctx_fini(&ww);
intel_runtime_pm_put(rpm, wakeref); return i915_error_to_vmf_fault(ret);
}
/* As this is primarily for debugging, let's focus on simplicity */
vaddr = i915_gem_object_pin_map(obj, I915_MAP_FORCE_WC); if (IS_ERR(vaddr)) {
err = PTR_ERR(vaddr); goto out;
}
/* * It is vital that we remove the page mapping if we have mapped a tiled * object through the GTT and then lose the fence register due to * resource pressure. Similarly if the object has been moved out of the * aperture, than pages mapped into userspace must be revoked. Removing the * mapping will then trigger a page fault on the next user access, allowing * fixup by vm_fault_gtt().
*/ void i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj)
{ struct drm_i915_private *i915 = to_i915(obj->base.dev);
intel_wakeref_t wakeref;
/* * Serialisation between user GTT access and our code depends upon * revoking the CPU's PTE whilst the mutex is held. The next user * pagefault then has to wait until we release the mutex. * * Note that RPM complicates somewhat by adding an additional * requirement that operations to the GGTT be made holding the RPM * wakeref.
*/
wakeref = intel_runtime_pm_get(&i915->runtime_pm);
mutex_lock(&to_gt(i915)->ggtt->vm.mutex);
if (!obj->userfault_count) goto out;
__i915_gem_object_release_mmap_gtt(obj);
/* * Ensure that the CPU's PTE are revoked and there are not outstanding * memory transactions from userspace before we return. The TLB * flushing implied above by changing the PTE above *should* be * sufficient, an extra barrier here just provides us with a bit * of paranoid documentation about our requirement to serialise * memory writes before touching registers / GSM.
*/
wmb();
/* * We have exclusive access here via runtime suspend. All other callers * must first grab the rpm wakeref.
*/
GEM_BUG_ON(!obj->userfault_count);
list_del(&obj->userfault_link);
obj->userfault_count = 0;
}
/* Attempt to reap some mmap space from dead objects */
err = intel_gt_retire_requests_timeout(to_gt(i915), MAX_SCHEDULE_TIMEOUT,
NULL); if (err) goto err;
/** * i915_gem_mmap_offset_ioctl - prepare an object for GTT mmap'ing * @dev: DRM device * @data: GTT mapping ioctl data * @file: GEM object info * * Simply returns the fake offset to userspace so it can mmap it. * The mmap call will end up in drm_gem_mmap(), which will set things * up so we can get faults in the handler above. * * The fault handler will take care of binding the object into the GTT * (since it may have been evicted to make room for something), allocating * a fence register, and mapping the appropriate aperture address into * userspace.
*/ int
i915_gem_mmap_offset_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
{ struct drm_i915_private *i915 = to_i915(dev); struct drm_i915_gem_mmap_offset *args = data; enum i915_mmap_type type; int err;
/* * Historically we failed to check args.pad and args.offset * and so we cannot use those fields for user input and we cannot * add -EINVAL for them as the ABI is fixed, i.e. old userspace * may be feeding in garbage in those fields. * * if (args->pad) return -EINVAL; is verbotten!
*/
err = i915_user_extensions(u64_to_user_ptr(args->extensions),
NULL, 0, NULL); if (err) return err;
switch (args->flags) { case I915_MMAP_OFFSET_GTT: if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt)) return -ENODEV;
type = I915_MMAP_TYPE_GTT; break;
case I915_MMAP_OFFSET_WC: if (!pat_enabled()) return -ENODEV;
type = I915_MMAP_TYPE_WC; break;
case I915_MMAP_OFFSET_WB:
type = I915_MMAP_TYPE_WB; break;
case I915_MMAP_OFFSET_UC: if (!pat_enabled()) return -ENODEV;
type = I915_MMAP_TYPE_UC; break;
case I915_MMAP_OFFSET_FIXED:
type = I915_MMAP_TYPE_FIXED; break;
/* * We keep the ref on mmo->obj, not vm_file, but we require * vma->vm_file->f_mapping, see vma_link(), for later revocation. * Our userspace is accustomed to having per-file resource cleanup * (i.e. contexts, objects and requests) on their close(fd), which * requires avoiding extraneous references to their filp, hence why * we prefer to use an anonymous file for their mmaps.
*/
vma_set_file(vma, anon); /* Drop the initial creation reference, the vma is now holding one. */
fput(anon);
/* * This overcomes the limitation in drm_gem_mmap's assignment of a * drm_gem_object as the vma->vm_private_data. Since we need to * be able to resolve multiple mmap offsets which could be tied * to a single gem object.
*/ int i915_gem_mmap(struct file *filp, struct vm_area_struct *vma)
{ struct drm_vma_offset_node *node; struct drm_file *priv = filp->private_data; struct drm_device *dev = priv->minor->dev; struct drm_i915_gem_object *obj = NULL; struct i915_mmap_offset *mmo = NULL;
if (drm_dev_is_unplugged(dev)) return -ENODEV;
rcu_read_lock();
drm_vma_offset_lock_lookup(dev->vma_offset_manager);
node = drm_vma_offset_lookup_locked(dev->vma_offset_manager,
vma->vm_pgoff,
vma_pages(vma)); if (node && drm_vma_node_is_allowed(node, priv)) { /* * Skip 0-refcnted objects as it is in the process of being * destroyed and will be invalid when the vma manager lock * is released.
*/ if (!node->driver_private) {
mmo = container_of(node, struct i915_mmap_offset, vma_node);
obj = i915_gem_object_get_rcu(mmo->obj);
/* * When we install vm_ops for mmap we are too late for * the vm_ops->open() which increases the ref_count of * this obj and then it gets decreased by the vm_ops->close(). * To balance this increase the obj ref_count here.
*/
obj = i915_gem_object_get(obj); return i915_gem_object_mmap(obj, mmo, vma);
}
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.
