static __always_inline void amd_iommu_atomic128_set(__int128 *ptr, __int128 val)
{ /* * Note: * We use arch_cmpxchg128_local() because: * - Need cmpxchg16b instruction mainly for 128-bit store to DTE * (not necessary for cmpxchg since this function is already * protected by a spin_lock for this DTE). * - Neither need LOCK_PREFIX nor try loop because of the spin_lock.
*/
arch_cmpxchg128_local(ptr, *ptr, val);
}
old.data128[1] = ptr->data128[1]; /* * Preserve DTE_DATA2_INTR_MASK. This needs to be * done here since it requires to be inside * spin_lock(&dev_data->dte_lock) context.
*/
new->data[2] &= ~DTE_DATA2_INTR_MASK;
new->data[2] |= old.data[2] & DTE_DATA2_INTR_MASK;
/* * Note: * IOMMU reads the entire Device Table entry in a single 256-bit transaction * but the driver is programming DTE using 2 128-bit cmpxchg. So, the driver * need to ensure the following: * - DTE[V|GV] bit is being written last when setting. * - DTE[V|GV] bit is being written first when clearing. * * This function is used only by code, which updates DMA translation part of the DTE. * So, only consider control bits related to DMA when updating the entry.
*/ staticvoid update_dte256(struct amd_iommu *iommu, struct iommu_dev_data *dev_data, struct dev_table_entry *new)
{ unsignedlong flags; struct dev_table_entry *dev_table = get_dev_table(iommu); struct dev_table_entry *ptr = &dev_table[dev_data->devid];
spin_lock_irqsave(&dev_data->dte_lock, flags);
if (!(ptr->data[0] & DTE_FLAG_V)) { /* Existing DTE is not valid. */
write_dte_upper128(ptr, new);
write_dte_lower128(ptr, new);
iommu_flush_dte_sync(iommu, dev_data->devid);
} elseif (!(new->data[0] & DTE_FLAG_V)) { /* Existing DTE is valid. New DTE is not valid. */
write_dte_lower128(ptr, new);
write_dte_upper128(ptr, new);
iommu_flush_dte_sync(iommu, dev_data->devid);
} elseif (!FIELD_GET(DTE_FLAG_GV, ptr->data[0])) { /* * Both DTEs are valid. * Existing DTE has no guest page table.
*/
write_dte_upper128(ptr, new);
write_dte_lower128(ptr, new);
iommu_flush_dte_sync(iommu, dev_data->devid);
} elseif (!FIELD_GET(DTE_FLAG_GV, new->data[0])) { /* * Both DTEs are valid. * Existing DTE has guest page table, * new DTE has no guest page table,
*/
write_dte_lower128(ptr, new);
write_dte_upper128(ptr, new);
iommu_flush_dte_sync(iommu, dev_data->devid);
} elseif (FIELD_GET(DTE_GPT_LEVEL_MASK, ptr->data[2]) !=
FIELD_GET(DTE_GPT_LEVEL_MASK, new->data[2])) { /* * Both DTEs are valid and have guest page table, * but have different number of levels. So, we need * to upadte both upper and lower 128-bit value, which * require disabling and flushing.
*/ struct dev_table_entry clear = {};
/* First disable DTE */
write_dte_lower128(ptr, &clear);
iommu_flush_dte_sync(iommu, dev_data->devid);
/* Then update DTE */
write_dte_upper128(ptr, new);
write_dte_lower128(ptr, new);
iommu_flush_dte_sync(iommu, dev_data->devid);
} else { /* * Both DTEs are valid and have guest page table, * and same number of levels. We just need to only * update the lower 128-bit. So no need to disable DTE.
*/
write_dte_lower128(ptr, new);
}
/* * We cannot support PASID w/ existing v1 page table in the same domain * since it will be nested. However, existing domain w/ v2 page table * or passthrough mode can be used for PASID.
*/ staticinlinebool pdom_is_sva_capable(struct protection_domain *pdom)
{ return pdom_is_v2_pgtbl_mode(pdom) || pdom_is_in_pt_mode(pdom);
}
/* Writes the specific IOMMU for a device into the PCI segment rlookup table */ void amd_iommu_set_rlookup_table(struct amd_iommu *iommu, u16 devid)
{ struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
iommu = rlookup_amd_iommu(&pdev->dev); if (!iommu) return 0;
/* Copy the data from pdev */
dev_data = dev_iommu_priv_get(&pdev->dev); if (!dev_data) {
pr_err("%s : Failed to get dev_data for 0x%x\n", __func__, devid);
ret = -EINVAL; goto out;
}
get_dte256(iommu, dev_data, &new);
/* Setup alias */
alias_data = find_dev_data(iommu, alias); if (!alias_data) {
pr_err("%s : Failed to get alias dev_data for 0x%x\n", __func__, alias);
ret = -EINVAL; goto out;
}
update_dte256(iommu, alias_data, &new);
if (!dev_is_pci(dev)) return;
pdev = to_pci_dev(dev);
/* * The IVRS alias stored in the alias table may not be * part of the PCI DMA aliases if it's bus differs * from the original device.
*/
clone_alias(pdev, iommu->pci_seg->alias_table[pci_dev_id(pdev)], NULL);
/* For ACPI HID devices, there are no aliases */ if (!dev_is_pci(dev)) return;
/* * Add the IVRS alias to the pci aliases if it is on the same * bus. The IVRS table may know about a quirk that we don't.
*/
ivrs_alias = pci_seg->alias_table[pci_dev_id(pdev)]; if (ivrs_alias != pci_dev_id(pdev) &&
PCI_BUS_NUM(ivrs_alias) == pdev->bus->number)
pci_add_dma_alias(pdev, ivrs_alias & 0xff, 1);
if (dev_data == NULL) {
dev_data = alloc_dev_data(iommu, devid); if (!dev_data) return NULL;
if (translation_pre_enabled(iommu))
dev_data->defer_attach = true;
}
return dev_data;
}
/* * Find or create an IOMMU group for a acpihid device.
*/ staticstruct iommu_group *acpihid_device_group(struct device *dev)
{ struct acpihid_map_entry *p, *entry = NULL; int devid;
devid = get_acpihid_device_id(dev, &entry); if (devid < 0) return ERR_PTR(devid);
if (dev_data->ats_enabled) {
pci_disable_ats(pdev);
dev_data->ats_enabled = 0;
}
}
staticinlineint pdev_enable_cap_pri(struct pci_dev *pdev)
{ struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); int ret = -EINVAL;
if (dev_data->pri_enabled) return 0;
if (!dev_data->ats_enabled) return 0;
if (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) { /* * First reset the PRI state of the device. * FIXME: Hardcode number of outstanding requests for now
*/ if (!pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32)) {
dev_data->pri_enabled = 1;
dev_data->pri_tlp = pci_prg_resp_pasid_required(pdev);
if (dev_data->pri_enabled) {
pci_disable_pri(pdev);
dev_data->pri_enabled = 0;
}
}
staticinlineint pdev_enable_cap_pasid(struct pci_dev *pdev)
{ struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); int ret = -EINVAL;
if (dev_data->pasid_enabled) return 0;
if (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP) { /* Only allow access to user-accessible pages */
ret = pci_enable_pasid(pdev, 0); if (!ret)
dev_data->pasid_enabled = 1;
}
/* * This function checks if the driver got a valid device from the caller to * avoid dereferencing invalid pointers.
*/ staticbool check_device(struct device *dev)
{ struct amd_iommu_pci_seg *pci_seg; struct amd_iommu *iommu; int devid, sbdf;
/* * The dev_iommu_priv_set() needes to be called before setup_aliases. * Otherwise, subsequent call to dev_iommu_priv_get() will fail.
*/
dev_iommu_priv_set(dev, dev_data);
setup_aliases(iommu, dev);
/* * By default we use passthrough mode for IOMMUv2 capable device. * But if amd_iommu=force_isolation is set (e.g. to debug DMA to * invalid address), we ignore the capability for the device so * it'll be forced to go into translation mode.
*/ if ((iommu_default_passthrough() || !amd_iommu_force_isolation) &&
dev_is_pci(dev) && amd_iommu_gt_ppr_supported()) {
dev_data->flags = pdev_get_caps(to_pci_dev(dev));
}
if (dev_data) { /* * If this is a DMA fault (for which the I(nterrupt) * bit will be unset), allow report_iommu_fault() to * prevent logging it.
*/ if (IS_IOMMU_MEM_TRANSACTION(flags)) { /* Device not attached to domain properly */ if (dev_data->domain == NULL) {
pr_err_ratelimited("Event logged [Device not attached to domain properly]\n");
pr_err_ratelimited(" device=%04x:%02x:%02x.%x domain=0x%04x\n",
iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid),
PCI_FUNC(devid), domain_id); goto out;
}
if (type == 0) { /* Did we hit the erratum? */ if (++count == LOOP_TIMEOUT) {
pr_err("No event written to event log\n"); return;
}
udelay(1); goto retry;
}
/* * To detect the hardware errata 732 we need to clear the * entry back to zero. This issue does not exist on SNP * enabled system. Also this buffer is not writeable on * SNP enabled system.
*/ if (!amd_iommu_snp_en)
memset(__evt, 0, 4 * sizeof(u32));
}
int amd_iommu_register_ga_log_notifier(int (*notifier)(u32))
{
iommu_ga_log_notifier = notifier;
/* * Ensure all in-flight IRQ handlers run to completion before returning * to the caller, e.g. to ensure module code isn't unloaded while it's * being executed in the IRQ handler.
*/ if (!notifier)
synchronize_rcu();
if ((status & overflow_mask) && overflow_handler)
overflow_handler(iommu);
/* * Hardware bug: ERBT1312 * When re-enabling interrupt (by writing 1 * to clear the bit), the hardware might also try to set * the interrupt bit in the event status register. * In this scenario, the bit will be set, and disable * subsequent interrupts. * * Workaround: The IOMMU driver should read back the * status register and check if the interrupt bits are cleared. * If not, driver will need to go through the interrupt handler * again and re-clear the bits
*/
status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
}
}
/* * Builds an invalidation address which is suitable for one page or multiple * pages. Sets the size bit (S) as needed is more than one page is flushed.
*/ staticinline u64 build_inv_address(u64 address, size_t size)
{
u64 pages, end, msb_diff;
/* * msb_diff would hold the index of the most significant bit that * flipped between the start and end.
*/
msb_diff = fls64(end ^ address) - 1;
/* * Bits 63:52 are sign extended. If for some reason bit 51 is different * between the start and the end, invalidate everything.
*/ if (unlikely(msb_diff > 51)) {
address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
} else { /* * The msb-bit must be clear on the address. Just set all the * lower bits.
*/
address |= (1ull << msb_diff) - 1;
}
/* Clear bits 11:0 */
address &= PAGE_MASK;
/* Set the size bit - we flush more than one 4kb page */ return address | CMD_INV_IOMMU_PAGES_SIZE_MASK;
}
/* * Writes the command to the IOMMUs command buffer and informs the * hardware about the new command.
*/ staticint __iommu_queue_command_sync(struct amd_iommu *iommu, struct iommu_cmd *cmd, bool sync)
{ unsignedint count = 0;
u32 left, next_tail;
if (left <= 0x20) { /* Skip udelay() the first time around */ if (count++) { if (count == LOOP_TIMEOUT) {
pr_err("Command buffer timeout\n"); return -EIO;
}
udelay(1);
}
/* Update head and recheck remaining space */
iommu->cmd_buf_head = readl(iommu->mmio_base +
MMIO_CMD_HEAD_OFFSET);
goto again;
}
copy_cmd_to_buffer(iommu, cmd);
/* Do we need to make sure all commands are processed? */
iommu->need_sync = sync;
/* * This function queues a completion wait command into the command * buffer of an IOMMU
*/ staticint iommu_completion_wait(struct amd_iommu *iommu)
{ struct iommu_cmd cmd; unsignedlong flags; int ret;
u64 data;
if (!iommu->need_sync) return 0;
data = atomic64_inc_return(&iommu->cmd_sem_val);
build_completion_wait(&cmd, iommu, data);
raw_spin_lock_irqsave(&iommu->lock, flags);
ret = __iommu_queue_command_sync(iommu, &cmd, false); if (ret) goto out_unlock;
/* * Devices of this domain are behind this IOMMU * We need to wait for completion of all commands.
*/
xa_for_each(&domain->iommu_array, i, pdom_iommu_info)
iommu_completion_wait(pdom_iommu_info->iommu);
}
for (devid = 0; devid <= last_bdf; ++devid)
iommu_flush_dte(iommu, devid);
iommu_completion_wait(iommu);
}
/* * This function uses heavy locking and may disable irqs for some time. But * this is no issue because it is only called during resume.
*/ staticvoid amd_iommu_flush_tlb_all(struct amd_iommu *iommu)
{
u32 dom_id;
u16 last_bdf = iommu->pci_seg->last_bdf;
/* * Command send function for invalidating a device table entry
*/ staticint device_flush_dte(struct iommu_dev_data *dev_data)
{ struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); struct pci_dev *pdev = NULL; struct amd_iommu_pci_seg *pci_seg;
u16 alias; int ret;
if (dev_is_pci(dev_data->dev))
pdev = to_pci_dev(dev_data->dev);
if (pdev)
ret = pci_for_each_dma_alias(pdev,
device_flush_dte_alias, iommu); else
ret = iommu_flush_dte(iommu, dev_data->devid); if (ret) return ret;
pci_seg = iommu->pci_seg;
alias = pci_seg->alias_table[dev_data->devid]; if (alias != dev_data->devid) {
ret = iommu_flush_dte(iommu, alias); if (ret) return ret;
}
if (dev_data->ats_enabled) { /* Invalidate the entire contents of an IOTLB */
ret = device_flush_iotlb(dev_data, 0, ~0UL,
IOMMU_NO_PASID, false);
}
return ret;
}
staticint domain_flush_pages_v2(struct protection_domain *pdom,
u64 address, size_t size)
{ struct iommu_dev_data *dev_data; struct iommu_cmd cmd; int ret = 0;
xa_for_each(&pdom->iommu_array, i, pdom_iommu_info) { /* * Devices of this domain are behind this IOMMU * We need a TLB flush
*/
ret |= iommu_queue_command(pdom_iommu_info->iommu, &cmd);
}
return ret;
}
/* * TLB invalidation function which is called from the mapping functions. * It flushes range of PTEs of the domain.
*/ staticvoid __domain_flush_pages(struct protection_domain *domain,
u64 address, size_t size)
{ struct iommu_dev_data *dev_data; int ret = 0;
ioasid_t pasid = IOMMU_NO_PASID; bool gn = false;
lockdep_assert_held(&domain->lock);
if (pdom_is_v2_pgtbl_mode(domain)) {
gn = true;
ret = domain_flush_pages_v2(domain, address, size);
} else {
ret = domain_flush_pages_v1(domain, address, size);
}
if (likely(!amd_iommu_np_cache)) {
__domain_flush_pages(domain, address, size);
/* Wait until IOMMU TLB and all device IOTLB flushes are complete */
domain_flush_complete(domain);
return;
}
/* * When NpCache is on, we infer that we run in a VM and use a vIOMMU. * In such setups it is best to avoid flushes of ranges which are not * naturally aligned, since it would lead to flushes of unmodified * PTEs. Such flushes would require the hypervisor to do more work than * necessary. Therefore, perform repeated flushes of aligned ranges * until you cover the range. Each iteration flushes the smaller * between the natural alignment of the address that we flush and the * greatest naturally aligned region that fits in the range.
*/ while (size != 0) { int addr_alignment = __ffs(address); int size_alignment = __fls(size); int min_alignment;
size_t flush_size;
/* * size is always non-zero, but address might be zero, causing * addr_alignment to be negative. As the casting of the * argument in __ffs(address) to long might trim the high bits * of the address on x86-32, cast to long when doing the check.
*/ if (likely((unsignedlong)address != 0))
min_alignment = min(addr_alignment, size_alignment); else
min_alignment = size_alignment;
/* Wait until IOMMU TLB and all device IOTLB flushes are complete */
domain_flush_complete(domain);
}
/* Flush the whole IO/TLB for a given protection domain - including PDE */ staticvoid amd_iommu_domain_flush_all(struct protection_domain *domain)
{
amd_iommu_domain_flush_pages(domain, 0,
CMD_INV_IOMMU_ALL_PAGES_ADDRESS);
}
/* Flush the not present cache if it exists */ staticvoid domain_flush_np_cache(struct protection_domain *domain,
dma_addr_t iova, size_t size)
{ if (unlikely(amd_iommu_np_cache)) { unsignedlong flags;
/* * This function flushes the DTEs for all devices in domain
*/ void amd_iommu_update_and_flush_device_table(struct protection_domain *domain)
{ struct iommu_dev_data *dev_data;
/**************************************************************************** * * The next functions belong to the domain allocation. A domain is * allocated for every IOMMU as the default domain. If device isolation * is enabled, every device get its own domain. The most important thing * about domains is the page table mapping the DMA address space they * contain. *
****************************************************************************/
/* * Number of GCR3 table levels required. Level must be 4-Kbyte * page and can contain up to 512 entries.
*/ staticint get_gcr3_levels(int pasids)
{ int levels;
staticint setup_gcr3_table(struct gcr3_tbl_info *gcr3_info, struct amd_iommu *iommu, int pasids)
{ int levels = get_gcr3_levels(pasids); int nid = iommu ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE; int domid;
if (levels > amd_iommu_max_glx_val) return -EINVAL;
if (gcr3_info->gcr3_tbl) return -EBUSY;
/* Allocate per device domain ID */
domid = pdom_id_alloc(); if (domid <= 0) return -ENOSPC;
gcr3_info->domid = domid;
int amd_iommu_set_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid, unsignedlong gcr3)
{ struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; int ret;
iommu_group_mutex_assert(dev_data->dev);
ret = update_gcr3(dev_data, pasid, gcr3, true); if (ret) return ret;
gcr3_info->pasid_cnt++; return ret;
}
int amd_iommu_clear_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid)
{ struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; int ret;
iommu_group_mutex_assert(dev_data->dev);
ret = update_gcr3(dev_data, pasid, 0, false); if (ret) return ret;
gcr3_info->pasid_cnt--; return ret;
}
staticvoid make_clear_dte(struct iommu_dev_data *dev_data, struct dev_table_entry *ptr, struct dev_table_entry *new)
{ /* All existing DTE must have V bit set */
new->data128[0] = DTE_FLAG_V;
new->data128[1] = 0;
}
/* * Note: * The old value for GCR3 table and GPT have been cleared from caller.
*/ staticvoid set_dte_gcr3_table(struct amd_iommu *iommu, struct iommu_dev_data *dev_data, struct dev_table_entry *target)
{ struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;
u64 gcr3;
/* * When SNP is enabled, we can only support TV=1 with non-zero domain ID. * This is prevented by the SNP-enable and IOMMU_DOMAIN_IDENTITY check in * do_iommu_domain_alloc().
*/
WARN_ON(amd_iommu_snp_en && (domid == 0)); new.data[0] |= DTE_FLAG_TV;
if (dev_data->ppr) new.data[0] |= 1ULL << DEV_ENTRY_PPR;
if (domain->dirty_tracking) new.data[0] |= DTE_FLAG_HAD;
if (dev_data->ats_enabled) new.data[1] |= DTE_FLAG_IOTLB;
/* * Restore cached persistent DTE bits, which can be set by information * in IVRS table. See set_dev_entry_from_acpi().
*/
initial_dte = amd_iommu_get_ivhd_dte_flags(iommu->pci_seg->id, dev_data->devid); if (initial_dte) { new.data128[0] |= initial_dte->data128[0]; new.data128[1] |= initial_dte->data128[1];
}
set_dte_gcr3_table(iommu, dev_data, &new);
update_dte256(iommu, dev_data, &new);
/* * A kdump kernel might be replacing a domain ID that was copied from * the previous kernel--if so, it needs to flush the translation cache * entries for the old domain ID that is being overwritten
*/ if (old_domid) {
amd_iommu_flush_tlb_domid(iommu, old_domid);
}
}
/* * Clear DMA-remap related flags to block all DMA (blockeded domain)
*/ staticvoid clear_dte_entry(struct amd_iommu *iommu, struct iommu_dev_data *dev_data)
{ struct dev_table_entry new = {}; struct dev_table_entry *dte = &get_dev_table(iommu)[dev_data->devid];
/* * If domain is SVA capable then initialize GCR3 table. Also if domain is * in v2 page table mode then update GCR3[0].
*/ staticint init_gcr3_table(struct iommu_dev_data *dev_data, struct protection_domain *pdom)
{ struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); int max_pasids = dev_data->max_pasids; int ret = 0;
/* * If domain is in pt mode then setup GCR3 table only if device * is PASID capable
*/ if (pdom_is_in_pt_mode(pdom) && !pdev_pasid_supported(dev_data)) return ret;
/* * By default, setup GCR3 table to support MAX PASIDs * supported by the device/IOMMU.
*/
ret = setup_gcr3_table(&dev_data->gcr3_info, iommu,
max_pasids > 0 ? max_pasids : 1); if (ret) return ret;
/* Setup GCR3[0] only if domain is setup with v2 page table mode */ if (!pdom_is_v2_pgtbl_mode(pdom)) return ret;
ret = update_gcr3(dev_data, 0, iommu_virt_to_phys(pdom->iop.pgd), true); if (ret)
free_gcr3_table(&dev_data->gcr3_info);
pdom_iommu_info->refcnt--; if (pdom_iommu_info->refcnt == 0) {
xa_erase(&pdom->iommu_array, iommu->index);
kfree(pdom_iommu_info);
}
spin_unlock_irqrestore(&pdom->lock, flags);
}
/* * If a device is not yet associated with a domain, this function makes the * device visible in the domain
*/ staticint attach_device(struct device *dev, struct protection_domain *domain)
{ struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); struct pci_dev *pdev; unsignedlong flags; int ret = 0;
mutex_lock(&dev_data->mutex);
if (dev_data->domain != NULL) {
ret = -EBUSY; goto out;
}
/* Do reference counting */
ret = pdom_attach_iommu(iommu, domain); if (ret) goto out;
/* Setup GCR3 table */ if (pdom_is_sva_capable(domain)) {
ret = init_gcr3_table(dev_data, domain); if (ret) {
pdom_detach_iommu(iommu, domain); goto out;
}
}
/* * Device can continue to function even if IOPF * enablement failed. Hence in error path just * disable device PRI support.
*/ if (amd_iommu_iopf_add_device(iommu, dev_data))
pdev_disable_cap_pri(pdev);
} elseif (pdev) {
pdev_enable_cap_ats(pdev);
}
/* * First check if the device is still attached. It might already * be detached from its domain because the generic * iommu_detach_group code detached it and we try again here in * our alias handling.
*/ if (WARN_ON(!dev_data->domain)) goto out;
if (dev_is_pci(dev))
pdev_disable_caps(to_pci_dev(dev));
/* Clear DTE and flush the entry */
dev_update_dte(dev_data, false);
/* Flush IOTLB and wait for the flushes to finish */
spin_lock_irqsave(&domain->lock, flags);
amd_iommu_domain_flush_all(domain);
list_del(&dev_data->list);
spin_unlock_irqrestore(&domain->lock, flags);
/* Clear GCR3 table */ if (pdom_is_sva_capable(domain))
destroy_gcr3_table(dev_data, domain);
/* Update data structures */
dev_data->domain = NULL;
/* decrease reference counters - needs to happen after the flushes */
pdom_detach_iommu(iommu, domain);
/* * If IOMMU and device supports PASID then it will contain max * supported PASIDs, else it will be zero.
*/
dev_data = dev_iommu_priv_get(dev); if (amd_iommu_pasid_supported() && dev_is_pci(dev) &&
pdev_pasid_supported(dev_data)) {
dev_data->max_pasids = min_t(u32, iommu->iommu.max_pasids,
pci_max_pasids(to_pci_dev(dev)));
}
if (amd_iommu_pgtable == PD_MODE_NONE) {
pr_warn_once("%s: DMA translation not supported by iommu.\n",
__func__);
iommu_dev = ERR_PTR(-ENODEV); goto out_err;
}
out_err:
iommu_completion_wait(iommu);
if (FEATURE_NUM_INT_REMAP_SUP_2K(amd_iommu_efr2))
dev_data->max_irqs = MAX_IRQS_PER_TABLE_2K; else
dev_data->max_irqs = MAX_IRQS_PER_TABLE_512;
if (dev_is_pci(dev))
pci_prepare_ats(to_pci_dev(dev), PAGE_SHIFT);
/* * We keep dev_data around for unplugged devices and reuse it when the * device is re-plugged - not doing so would introduce a ton of races.
*/
}
staticstruct iommu_group *amd_iommu_device_group(struct device *dev)
{ if (dev_is_pci(dev)) return pci_device_group(dev);
return acpihid_device_group(dev);
}
/***************************************************************************** * * The following functions belong to the exported interface of AMD IOMMU * * This interface allows access to lower level functions of the IOMMU * like protection domain handling and assignement of devices to domains * which is not possible with the dma_ops interface. *
*****************************************************************************/
/* * V2 with 4/5 level page table. Note that "2.2.6.5 AMD64 4-Kbyte Page * Translation" shows that the V2 table sign extends the top of the * address space creating a reserved region in the middle of the * translation, just like the CPU does. Further Vasant says the docs are * incomplete and this only applies to non-zero PASIDs. If the AMDv2 * page table is assigned to the 0 PASID then there is no sign extension * check. * * Since the IOMMU must have a fixed geometry, and the core code does * not understand sign extended addressing, we have to chop off the high * bit to get consistent behavior with attachments of the domain to any * PASID.
*/ return ((1ULL << (PM_LEVEL_SHIFT(amd_iommu_gpt_level) - 1)) - 1);
}
staticbool amd_iommu_hd_support(struct amd_iommu *iommu)
{ if (amd_iommu_hatdis) returnfalse;
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.
