// SPDX-License-Identifier: GPL-2.0 OR MIT /* * Copyright 2014-2022 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE.
*/
/* * kfd_locked is used to lock the kfd driver during suspend or reset * once locked, kfd driver will stop any further GPU execution. * create process (open) will return -EAGAIN.
*/ staticint kfd_locked;
/* * flush_work ensures that there are no outstanding * work-queue items that will access interrupt_ring. New work items * can't be created because we stopped interrupt handling above.
*/
flush_workqueue(kfd->ih_wq);
destroy_workqueue(kfd->ih_wq);
for (i = 0; i < num_nodes; i++) {
knode = kfd->nodes[i];
device_queue_manager_uninit(knode->dqm);
kfd_interrupt_exit(knode);
kfd_topology_remove_device(knode); if (knode->gws)
amdgpu_amdkfd_free_gws(knode->adev, knode->gws);
kfree(knode);
kfd->nodes[i] = NULL;
}
}
staticvoid kfd_setup_interrupt_bitmap(struct kfd_node *node, unsignedint kfd_node_idx)
{ struct amdgpu_device *adev = node->adev;
uint32_t xcc_mask = node->xcc_mask;
uint32_t xcc, mapped_xcc; /* * Interrupt bitmap is setup for processing interrupts from * different XCDs and AIDs. * Interrupt bitmap is defined as follows: * 1. Bits 0-15 - correspond to the NodeId field. * Each bit corresponds to NodeId number. For example, if * a KFD node has interrupt bitmap set to 0x7, then this * KFD node will process interrupts with NodeId = 0, 1 and 2 * in the IH cookie. * 2. Bits 16-31 - unused. * * Please note that the kfd_node_idx argument passed to this * function is not related to NodeId field received in the * IH cookie. * * In CPX mode, a KFD node will process an interrupt if: * - the Node Id matches the corresponding bit set in * Bits 0-15. * - AND VMID reported in the interrupt lies within the * VMID range of the node.
*/
for_each_inst(xcc, xcc_mask) {
mapped_xcc = GET_INST(GC, xcc);
node->interrupt_bitmap |= (mapped_xcc % 2 ? 5 : 3) << (4 * (mapped_xcc / 2));
}
dev_info(kfd_device, "Node: %d, interrupt_bitmap: %x\n", kfd_node_idx,
node->interrupt_bitmap);
}
/* For multi-partition capable GPUs, we need special handling for VMIDs * depending on partition mode. * In CPX mode, the VMID range needs to be shared between XCDs. * Additionally, there are 13 VMIDs (3-15) available for KFD. To * divide them equally, we change starting VMID to 4 and not use * VMID 3. * If the VMID range changes for multi-partition capable GPUs, then * this code MUST be revisited.
*/ if (kfd->adev->xcp_mgr) {
partition_mode = amdgpu_xcp_query_partition_mode(kfd->adev->xcp_mgr,
AMDGPU_XCP_FL_LOCKED); if (partition_mode == AMDGPU_CPX_PARTITION_MODE &&
kfd->num_nodes != 1) {
vmid_num_kfd /= 2;
first_vmid_kfd = last_vmid_kfd + 1 - vmid_num_kfd*2;
}
}
/* Verify module parameters regarding mapped process number*/ if (hws_max_conc_proc >= 0)
max_proc_per_quantum = min((u32)hws_max_conc_proc, vmid_num_kfd); else
max_proc_per_quantum = vmid_num_kfd;
/* calculate max size of mqds needed for queues */
size = max_num_of_queues_per_device *
kfd->device_info.mqd_size_aligned;
/* * calculate max size of runlist packet. * There can be only 2 packets at once
*/
map_process_packet_size = KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 2) ? sizeof(struct pm4_mes_map_process_aldebaran) : sizeof(struct pm4_mes_map_process);
size += (KFD_MAX_NUM_OF_PROCESSES * map_process_packet_size +
max_num_of_queues_per_device * sizeof(struct pm4_mes_map_queues)
+ sizeof(struct pm4_mes_runlist)) * 2;
if (amdgpu_use_xgmi_p2p)
kfd->hive_id = kfd->adev->gmc.xgmi.hive_id;
/* * For multi-partition capable GPUs, the KFD abstracts all partitions * within a socket as xGMI connected in the topology so assign a unique * hive id per device based on the pci device location if device is in * PCIe mode.
*/ if (!kfd->hive_id && kfd->num_nodes > 1)
kfd->hive_id = pci_dev_id(kfd->adev->pdev);
kfd->noretry = kfd->adev->gmc.noretry;
kfd_cwsr_init(kfd);
dev_info(kfd_device, "Total number of KFD nodes to be created: %d\n",
kfd->num_nodes);
/* Allocate the KFD nodes */ for (i = 0, xcp_idx = 0; i < kfd->num_nodes; i++) {
node = kzalloc(sizeof(struct kfd_node), GFP_KERNEL); if (!node) goto node_alloc_error;
if (partition_mode == AMDGPU_CPX_PARTITION_MODE &&
kfd->num_nodes != 1) { /* For multi-partition capable GPUs and CPX mode, first * XCD gets VMID range 4-9 and second XCD gets VMID * range 10-15.
*/
int kgd2kfd_pre_reset(struct kfd_dev *kfd, struct amdgpu_reset_context *reset_context)
{ struct kfd_node *node; int i;
if (!kfd->init_complete) return 0;
for (i = 0; i < kfd->num_nodes; i++) {
node = kfd->nodes[i];
kfd_smi_event_update_gpu_reset(node, false, reset_context);
}
kgd2kfd_suspend(kfd, true);
for (i = 0; i < kfd->num_nodes; i++)
kfd_signal_reset_event(kfd->nodes[i]);
return 0;
}
/* * Fix me. KFD won't be able to resume existing process for now. * We will keep all existing process in a evicted state and * wait the process to be terminated.
*/
int kgd2kfd_post_reset(struct kfd_dev *kfd)
{ int ret; struct kfd_node *node; int i;
if (!kfd->init_complete) return 0;
for (i = 0; i < kfd->num_nodes; i++) {
ret = kfd_resume(kfd->nodes[i]); if (ret) return ret;
}
/* check reset/suspend lock */ if (kfd_locked > 0) returntrue;
if (kfd) return kfd->kfd_dev_lock > 0;
/* check lock on all cgroup accessible devices */ while (kfd_topology_enum_kfd_devices(id++, &dev) == 0) { if (!dev || kfd_devcgroup_check_permission(dev)) continue;
for (i = 0; i < kfd->num_nodes; i++) {
node = kfd->nodes[i];
node->dqm->ops.stop(node->dqm);
}
}
int kgd2kfd_resume(struct kfd_dev *kfd, bool resume_proc)
{ int ret, i;
if (!kfd->init_complete) return 0;
for (i = 0; i < kfd->num_nodes; i++) {
ret = kfd_resume(kfd->nodes[i]); if (ret) return ret;
}
if (resume_proc)
ret = kgd2kfd_resume_process(kfd);
return ret;
}
void kgd2kfd_suspend_process(struct kfd_dev *kfd)
{ if (!kfd->init_complete) return;
mutex_lock(&kfd_processes_mutex); /* For first KFD device suspend all the KFD processes */ if (++kfd_locked == 1)
kfd_suspend_all_processes();
mutex_unlock(&kfd_processes_mutex);
}
int kgd2kfd_resume_process(struct kfd_dev *kfd)
{ int ret = 0;
if (!kfd->init_complete) return 0;
mutex_lock(&kfd_processes_mutex); if (--kfd_locked == 0)
ret = kfd_resume_all_processes();
WARN_ONCE(kfd_locked < 0, "KFD suspend / resume ref. error");
mutex_unlock(&kfd_processes_mutex);
return ret;
}
staticint kfd_resume(struct kfd_node *node)
{ int err = 0;
err = node->dqm->ops.start(node->dqm); if (err)
dev_err(kfd_device, "Error starting queue manager for device %x:%x\n",
node->adev->pdev->vendor, node->adev->pdev->device);
return err;
}
/* This is called directly from KGD at ISR. */ void kgd2kfd_interrupt(struct kfd_dev *kfd, constvoid *ih_ring_entry)
{
uint32_t patched_ihre[KFD_MAX_RING_ENTRY_SIZE], i; bool is_patched = false; unsignedlong flags; struct kfd_node *node;
if (!kfd->init_complete) return;
if (kfd->device_info.ih_ring_entry_size > sizeof(patched_ihre)) {
dev_err_once(kfd_device, "Ring entry too small\n"); return;
}
for (i = 0; i < kfd->num_nodes; i++) { /* Race if another thread in b/w * kfd_cleanup_nodes and kfree(kfd), * when kfd->nodes[i] = NULL
*/ if (kfd->nodes[i])
node = kfd->nodes[i]; else return;
int kgd2kfd_quiesce_mm(struct mm_struct *mm, uint32_t trigger)
{ struct kfd_process *p; int r;
/* Because we are called from arbitrary context (workqueue) as opposed * to process context, kfd_process could attempt to exit while we are * running so the lookup function increments the process ref count.
*/
p = kfd_lookup_process_by_mm(mm); if (!p) return -ESRCH;
WARN(debug_evictions, "Evicting pid %d", p->lead_thread->pid);
r = kfd_process_evict_queues(p, trigger);
kfd_unref_process(p); return r;
}
int kgd2kfd_resume_mm(struct mm_struct *mm)
{ struct kfd_process *p; int r;
/* Because we are called from arbitrary context (workqueue) as opposed * to process context, kfd_process could attempt to exit while we are * running so the lookup function increments the process ref count.
*/
p = kfd_lookup_process_by_mm(mm); if (!p) return -ESRCH;
r = kfd_process_restore_queues(p);
kfd_unref_process(p); return r;
}
/** kgd2kfd_schedule_evict_and_restore_process - Schedules work queue that will * prepare for safe eviction of KFD BOs that belong to the specified * process. * * @mm: mm_struct that identifies the specified KFD process * @fence: eviction fence attached to KFD process BOs *
*/ int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm, struct dma_fence *fence)
{ struct kfd_process *p; unsignedlong active_time; unsignedlong delay_jiffies = msecs_to_jiffies(PROCESS_ACTIVE_TIME_MS);
if (!fence) return -EINVAL;
if (dma_fence_is_signaled(fence)) return 0;
p = kfd_lookup_process_by_mm(mm); if (!p) return -ENODEV;
if (fence->seqno == p->last_eviction_seqno) goto out;
p->last_eviction_seqno = fence->seqno;
/* Avoid KFD process starvation. Wait for at least * PROCESS_ACTIVE_TIME_MS before evicting the process again
*/
active_time = get_jiffies_64() - p->last_restore_timestamp; if (delay_jiffies > active_time)
delay_jiffies -= active_time; else
delay_jiffies = 0;
/* During process initialization eviction_work.dwork is initialized * to kfd_evict_bo_worker
*/
WARN(debug_evictions, "Scheduling eviction of pid %d in %ld jiffies",
p->lead_thread->pid, delay_jiffies);
schedule_delayed_work(&p->eviction_work, delay_jiffies);
out:
kfd_unref_process(p); return 0;
}
staticint kfd_gtt_sa_init(struct kfd_dev *kfd, unsignedint buf_size, unsignedint chunk_size)
{ if (WARN_ON(buf_size < chunk_size)) return -EINVAL; if (WARN_ON(buf_size == 0)) return -EINVAL; if (WARN_ON(chunk_size == 0)) return -EINVAL;
if (size > kfd->gtt_sa_num_of_chunks * kfd->gtt_sa_chunk_size) return -ENOMEM;
*mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_KERNEL); if (!(*mem_obj)) return -ENOMEM;
pr_debug("Allocated mem_obj = %p for size = %d\n", *mem_obj, size);
start_search = 0;
mutex_lock(&kfd->gtt_sa_lock);
kfd_gtt_restart_search: /* Find the first chunk that is free */
found = find_next_zero_bit(kfd->gtt_sa_bitmap,
kfd->gtt_sa_num_of_chunks,
start_search);
pr_debug("Found = %d\n", found);
/* If there wasn't any free chunk, bail out */ if (found == kfd->gtt_sa_num_of_chunks) goto kfd_gtt_no_free_chunk;
/* If we need only one chunk, mark it as allocated and get out */ if (size <= kfd->gtt_sa_chunk_size) {
pr_debug("Single bit\n");
__set_bit(found, kfd->gtt_sa_bitmap); goto kfd_gtt_out;
}
/* Otherwise, try to see if we have enough contiguous chunks */
cur_size = size - kfd->gtt_sa_chunk_size; do {
(*mem_obj)->range_end =
find_next_zero_bit(kfd->gtt_sa_bitmap,
kfd->gtt_sa_num_of_chunks, ++found); /* * If next free chunk is not contiguous than we need to * restart our search from the last free chunk we found (which * wasn't contiguous to the previous ones
*/ if ((*mem_obj)->range_end != found) {
start_search = found; goto kfd_gtt_restart_search;
}
/* * If we reached end of buffer, bail out with error
*/ if (found == kfd->gtt_sa_num_of_chunks) goto kfd_gtt_no_free_chunk;
/* Check if we don't need another chunk */ if (cur_size <= kfd->gtt_sa_chunk_size)
cur_size = 0; else
cur_size -= kfd->gtt_sa_chunk_size;
/* Mark the chunks as free */
bitmap_clear(kfd->gtt_sa_bitmap, mem_obj->range_start,
mem_obj->range_end - mem_obj->range_start + 1);
mutex_unlock(&kfd->gtt_sa_lock);
kfree(mem_obj); return 0;
}
void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd)
{ /* * TODO: Currently update SRAM ECC flag for first node. * This needs to be updated later when we can * identify SRAM ECC error on other nodes also.
*/ if (kfd)
atomic_inc(&kfd->nodes[0]->sram_ecc_flag);
}
staticbool kfd_compute_active(struct kfd_node *node)
{ if (atomic_read(&node->kfd->compute_profile)) returntrue; returnfalse;
}
void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint64_t throttle_bitmask)
{ /* * TODO: For now, raise the throttling event only on first node. * This will need to change after we are able to determine * which node raised the throttling event.
*/ if (kfd && kfd->init_complete)
kfd_smi_event_update_thermal_throttling(kfd->nodes[0],
throttle_bitmask);
}
/* kfd_get_num_sdma_engines returns the number of PCIe optimized SDMA and * kfd_get_num_xgmi_sdma_engines returns the number of XGMI SDMA. * When the device has more than two engines, we reserve two for PCIe to enable * full-duplex and the rest are used as XGMI.
*/ unsignedint kfd_get_num_sdma_engines(struct kfd_node *node)
{ /* If XGMI is not supported, all SDMA engines are PCIe */ if (!node->adev->gmc.xgmi.supported) return node->adev->sdma.num_instances/(int)node->kfd->num_nodes;
unsignedint kfd_get_num_xgmi_sdma_engines(struct kfd_node *node)
{ /* After reserved for PCIe, the rest of engines are XGMI */ return node->adev->sdma.num_instances/(int)node->kfd->num_nodes -
kfd_get_num_sdma_engines(node);
}
int kgd2kfd_check_and_lock_kfd(struct kfd_dev *kfd)
{ struct kfd_process *p; int r = 0, temp, idx;
mutex_lock(&kfd_processes_mutex);
/* kfd_processes_count is per kfd_dev, return -EBUSY without * further check
*/ if (!!atomic_read(&kfd->kfd_processes_count)) {
pr_debug("process_wq_release not finished\n");
r = -EBUSY; goto out;
}
if (hash_empty(kfd_processes_table) && !kfd_is_locked(kfd)) goto out;
/* fail under system reset/resume or kfd device is partition switching. */ if (kfd_is_locked(kfd)) {
r = -EBUSY; goto out;
}
/* * ensure all running processes are cgroup excluded from device before mode switch. * i.e. no pdd was created on the process socket.
*/
idx = srcu_read_lock(&kfd_processes_srcu);
hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) { int i;
for (i = 0; i < p->n_pdds; i++) { if (p->pdds[i]->dev->kfd != kfd) continue;
r = -EBUSY; goto proc_check_unlock;
}
}
proc_check_unlock:
srcu_read_unlock(&kfd_processes_srcu, idx);
out: if (!r)
++kfd->kfd_dev_lock;
mutex_unlock(&kfd_processes_mutex);
ret = node->dqm->ops.unhalt(node->dqm); if (ret)
dev_err(kfd_device, "Error in starting scheduler\n");
return ret;
}
int kgd2kfd_start_sched_all_nodes(struct kfd_dev *kfd)
{ struct kfd_node *node; int i, r;
if (!kfd->init_complete) return 0;
for (i = 0; i < kfd->num_nodes; i++) {
node = kfd->nodes[i];
r = node->dqm->ops.unhalt(node->dqm); if (r) {
dev_err(kfd_device, "Error in starting scheduler\n"); return r;
}
} return 0;
}
int kgd2kfd_stop_sched(struct kfd_dev *kfd, uint32_t node_id)
{ struct kfd_node *node;
/** * kgd2kfd_vmfault_fast_path() - KFD vm page fault interrupt handling fast path for gmc v9 * @adev: amdgpu device * @entry: vm fault interrupt vector * @retry_fault: if this is retry fault * * retry fault - * with CAM enabled, adev primary ring * | gmc_v9_0_process_interrupt() * adev soft_ring * | gmc_v9_0_process_interrupt() worker failed to recover page fault * KFD node ih_fifo * | KFD interrupt_wq worker * kfd_signal_vm_fault_event * * without CAM, adev primary ring1 * | gmc_v9_0_process_interrupt worker failed to recvoer page fault * KFD node ih_fifo * | KFD interrupt_wq worker * kfd_signal_vm_fault_event * * no-retry fault - * adev primary ring * | gmc_v9_0_process_interrupt() * KFD node ih_fifo * | KFD interrupt_wq worker * kfd_signal_vm_fault_event * * fast path - After kfd_signal_vm_fault_event, gmc_v9_0_process_interrupt drop the page fault * of same process, don't copy interrupt to KFD node ih_fifo. * With gdb debugger enabled, need convert the retry fault to no-retry fault for * debugger, cannot use the fast path. * * Return: * true - use the fast path to handle this fault * false - use normal path to handle it
*/ bool kgd2kfd_vmfault_fast_path(struct amdgpu_device *adev, struct amdgpu_iv_entry *entry, bool retry_fault)
{ struct kfd_process *p;
u32 cam_index;
if (entry->ih == &adev->irq.ih_soft || entry->ih == &adev->irq.ih1) {
p = kfd_lookup_process_by_pasid(entry->pasid, NULL); if (!p) returntrue;
if (p->gpu_page_fault && !p->debug_trap_enabled) { if (retry_fault && adev->irq.retry_cam_enabled) {
cam_index = entry->src_data[2] & 0x3ff;
WDOORBELL32(adev->irq.retry_cam_doorbell_index, cam_index);
}
kfd_unref_process(p); returntrue;
}
/* * This is the first page fault, set flag and then signal user space
*/
p->gpu_page_fault = true;
kfd_unref_process(p);
} returnfalse;
}
#ifdefined(CONFIG_DEBUG_FS)
/* This function will send a package to HIQ to hang the HWS * which will trigger a GPU reset and bring the HWS back to normal state
*/ int kfd_debugfs_hang_hws(struct kfd_node *dev)
{ if (dev->dqm->sched_policy != KFD_SCHED_POLICY_HWS) {
pr_err("HWS is not enabled"); return -EINVAL;
}
if (dev->kfd->shared_resources.enable_mes) {
dev_err(dev->adev->dev, "Inducing MES hang is not supported\n"); return -EINVAL;
}
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.
