--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c @@ -28,6 +28,7 @@ #include "amdgpu.h" #include "soc15_common.h" #include "amdgpu_mes_ctx.h" +#include "amdgpu_reset.h" #define AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS 1024 #define AMDGPU_ONE_DOORBELL_SIZE 8 @@ -100,6 +101,8 @@ return (total_hqd_mask & ~reserved_hqd_mask); } + +static void amdgpu_mes_userq_notify_unmap_work_handler(struct work_struct *work); int amdgpu_mes_init(struct amdgpu_device *adev) { @@ -123,6 +126,9 @@ spin_lock_init(&adev->mes.ring_lock[i]); adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK; + atomic_set(&adev->mes.userq_hw_queue_count, 0); + INIT_DELAYED_WORK(&adev->mes.userq_notify_unmap_work, + amdgpu_mes_userq_notify_unmap_work_handler); total_vmid_mask = (u32)((1UL << 16) - 1); reserved_vmid_mask = (u32)((1UL << adev->vm_manager.first_kfd_vmid) - 1); @@ -258,6 +264,8 @@ { int i; int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1; + + cancel_delayed_work_sync(&adev->mes.userq_notify_unmap_work); amdgpu_bo_free_kernel(&adev->mes.event_log_gpu_obj, &adev->mes.event_log_gpu_addr, @@ -801,6 +809,102 @@ return r; } +int amdgpu_mes_notify_unmap_queue(struct amdgpu_device *adev) +{ + struct mes_misc_op_input op_input = {0}; + int r; + + op_input.op = MES_MISC_OP_NOTIFY_WORK_ON_UNMAPPED_QUEUE; + + if (!adev->mes.funcs->misc_op) { + dev_err(adev->dev, "mes notify unmap queue is not supported!\n"); + r = -EINVAL; + goto error; + } + + amdgpu_mes_lock(&adev->mes); + r = adev->mes.funcs->misc_op(&adev->mes, &op_input); + amdgpu_mes_unlock(&adev->mes); + if (r) + dev_err(adev->dev, "failed to notify unmap queue.\n"); + +error: + return r; +} + +/* Interval for notifying MES of work on unmapped queues during oversubscription */ +#define AMDGPU_USERQ_UNMAP_NOTIFY_DELAY_US 50 + +static unsigned int amdgpu_mes_userq_hw_queue_num(struct amdgpu_device *adev) +{ + int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1; + unsigned int n = bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES); + int i; + + for (i = 0; i < num_xcc; i++) + n += bitmap_weight(adev->gfx.mec_bitmap[i].queue_bitmap, + AMDGPU_MAX_COMPUTE_QUEUES); + + return n; +} + +static void amdgpu_mes_userq_notify_unmap_work_handler(struct work_struct *work) +{ + struct amdgpu_mes *mes = container_of(work, struct amdgpu_mes, + userq_notify_unmap_work.work); + struct amdgpu_device *adev = mes->adev; + + /* MES is unusable during GPU recovery; the re-arm below retries later */ + if (down_read_trylock(&adev->reset_domain->sem)) { + amdgpu_mes_notify_unmap_queue(adev); + up_read(&adev->reset_domain->sem); + } + + /* Re-arm if still oversubscribed */ + if (atomic_read(&mes->userq_hw_queue_count) > + amdgpu_mes_userq_hw_queue_num(adev)) + queue_delayed_work(system_wq, &mes->userq_notify_unmap_work, + usecs_to_jiffies(AMDGPU_USERQ_UNMAP_NOTIFY_DELAY_US)); +} + +/* + * Called after a GFX11 usermode queue is successfully mapped to MES. + * Starts the periodic unmap-notify timer if this pushed the device into + * HW queue oversubscription. + */ +void amdgpu_mes_userq_queue_mapped(struct amdgpu_device *adev) +{ + if (amdgpu_sriov_vf(adev)) + return; + + if (!(amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0) && + amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(12, 0, 0))) + return; + + if (atomic_inc_return(&adev->mes.userq_hw_queue_count) > + amdgpu_mes_userq_hw_queue_num(adev)) + queue_delayed_work(system_wq, &adev->mes.userq_notify_unmap_work, + usecs_to_jiffies(AMDGPU_USERQ_UNMAP_NOTIFY_DELAY_US)); +} + +/* + * Called after a GFX11 usermode queue is unmapped from MES. Stops the + * periodic unmap-notify timer once oversubscription clears. + */ +void amdgpu_mes_userq_queue_unmapped(struct amdgpu_device *adev) +{ + if (amdgpu_sriov_vf(adev)) + return; + + if (!(amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0) && + amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(12, 0, 0))) + return; + + if (atomic_dec_return(&adev->mes.userq_hw_queue_count) <= + amdgpu_mes_userq_hw_queue_num(adev)) + cancel_delayed_work(&adev->mes.userq_notify_unmap_work); +} + #if defined(CONFIG_DEBUG_FS) static int amdgpu_debugfs_mes_event_log_show(struct seq_file *m, void *unused) --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h @@ -29,6 +29,7 @@ #include "amdgpu_gfx.h" #include "amdgpu_doorbell.h" #include +#include #define AMDGPU_MES_MAX_COMPUTE_PIPES 8 #define AMDGPU_MES_MAX_GFX_PIPES 2 @@ -88,6 +89,10 @@ uint32_t total_max_queue; uint32_t max_doorbell_slices; + + /* GFX11 usermode queue oversubscription notify timer */ + atomic_t userq_hw_queue_count; + struct delayed_work userq_notify_unmap_work; uint64_t default_process_quantum; uint64_t default_gang_quantum; @@ -332,6 +337,7 @@ MES_MISC_OP_WRM_REG_WR_WAIT, MES_MISC_OP_SET_SHADER_DEBUGGER, MES_MISC_OP_CHANGE_CONFIG, + MES_MISC_OP_NOTIFY_WORK_ON_UNMAPPED_QUEUE, }; struct mes_misc_op_input { @@ -538,5 +544,8 @@ bool amdgpu_mes_suspend_resume_all_supported(struct amdgpu_device *adev); int amdgpu_mes_update_enforce_isolation(struct amdgpu_device *adev); +int amdgpu_mes_notify_unmap_queue(struct amdgpu_device *adev); +void amdgpu_mes_userq_queue_mapped(struct amdgpu_device *adev); +void amdgpu_mes_userq_queue_unmapped(struct amdgpu_device *adev); #endif /* __AMDGPU_MES_H__ */ --- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c @@ -153,6 +153,8 @@ return r; } + amdgpu_mes_userq_queue_mapped(adev); + DRM_DEBUG_DRIVER("Queue (doorbell:%d) mapped successfully\n", userq_props->doorbell_index); return 0; } @@ -172,9 +174,13 @@ amdgpu_mes_lock(&adev->mes); r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input); amdgpu_mes_unlock(&adev->mes); - if (r) + if (r) { DRM_ERROR("Failed to unmap queue in HW, err (%d)\n", r); - return r; + return r; + } + + amdgpu_mes_userq_queue_unmapped(adev); + return 0; } static int mes_userq_create_ctx_space(struct amdgpu_userq_mgr *uq_mgr, --- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c @@ -662,6 +662,15 @@ misc_pkt.change_config.option.bits.limit_single_process = input->change_config.option.limit_single_process; break; + case MES_MISC_OP_NOTIFY_WORK_ON_UNMAPPED_QUEUE: + if ((mes->adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) < 0x70) { + dev_warn_once(mes->adev->dev, + "MES FW version must be larger than 0x70 to support notify work on unmapped queue.\n"); + return 0; + } + misc_pkt.opcode = MESAPI_MISC__NOTIFY_WORK_ON_UNMAPPED_QUEUE; + misc_pkt.queue_sch_level = AMD_PRIORITY_LEVEL_NORMAL; + break; default: drm_err(adev_to_drm(mes->adev), "unsupported misc op (%d)\n", input->op); @@ -721,7 +730,15 @@ mes_set_hw_res_pkt.use_different_vmid_compute = 1; mes_set_hw_res_pkt.enable_reg_active_poll = 1; mes_set_hw_res_pkt.enable_level_process_quantum_check = 1; - mes_set_hw_res_pkt.oversubscription_timer = 50; + /* + * Bare metal: disabled here, KFD arms a software replacement only + * while oversubscribed (see kfd_device_queue_manager.c). SR-IOV + * guests keep the firmware timer instead of the KFD-side one, since + * the notify round-trip is much slower under SR-IOV and would + * otherwise stall remapping queues. + */ + if (amdgpu_sriov_vf(adev)) + mes_set_hw_res_pkt.oversubscription_timer = 50; if (amdgpu_mes_log_enable) { mes_set_hw_res_pkt.enable_mes_event_int_logging = 1; --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -46,6 +46,9 @@ /* See unmap_queues_cpsch() */ #define USE_DEFAULT_GRACE_PERIOD 0xffffffff +/* Interval for notifying MES of work on unmapped queues during oversubscription */ +#define DQM_MES_UNMAP_NOTIFY_DELAY_US 50 + static int set_pasid_vmid_mapping(struct device_queue_manager *dqm, u32 pasid, unsigned int vmid); @@ -268,7 +271,22 @@ q->properties.doorbell_off); dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); kfd_hws_hang(dqm); - } + return r; + } + + /* + * GFX11: start notify timer only once when oversubscription begins. + * Skip under SR-IOV: MES round-trips are far slower there, and this + * notify call shares mes->mutex_hidden with add/remove_hw_queue, so + * a slow notify can stall remapping queues. SR-IOV guests keep MES's + * own firmware oversubscription timer instead. + */ + if (!amdgpu_sriov_vf(adev) && + KFD_GC_VERSION(dqm->dev) >= IP_VERSION(11, 0, 0) && + KFD_GC_VERSION(dqm->dev) < IP_VERSION(12, 0, 0) && + dqm->active_cp_queue_count > get_cp_queues_num(dqm)) + queue_delayed_work(system_wq, &dqm->notify_unmap_work, + usecs_to_jiffies(DQM_MES_UNMAP_NOTIFY_DELAY_US)); return r; } @@ -301,6 +319,13 @@ dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); kfd_hws_hang(dqm); } + + /* GFX11: stop notify timer when oversubscription clears */ + if (!r && + KFD_GC_VERSION(dqm->dev) >= IP_VERSION(11, 0, 0) && + KFD_GC_VERSION(dqm->dev) < IP_VERSION(12, 0, 0) && + dqm->active_cp_queue_count <= get_cp_queues_num(dqm)) + cancel_delayed_work(&dqm->notify_unmap_work); return r; } @@ -2916,6 +2941,20 @@ WARN(!mqd, "No hiq sdma mqd trunk to free"); amdgpu_amdkfd_free_kernel_mem(dev->adev, &mqd->mem); +} + +static void mes_notify_unmap_work_handler(struct work_struct *work) +{ + struct device_queue_manager *dqm = + container_of(work, struct device_queue_manager, + notify_unmap_work.work); + + amdgpu_mes_notify_unmap_queue((struct amdgpu_device *)dqm->dev->adev); + + /* Re-arm if still oversubscribed */ + if (READ_ONCE(dqm->active_cp_queue_count) > get_cp_queues_num(dqm)) + queue_delayed_work(system_wq, &dqm->notify_unmap_work, + usecs_to_jiffies(DQM_MES_UNMAP_NOTIFY_DELAY_US)); } struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev) @@ -3041,6 +3080,8 @@ if (!dqm->ops.initialize(dqm)) { init_waitqueue_head(&dqm->destroy_wait); + INIT_DELAYED_WORK(&dqm->notify_unmap_work, + mes_notify_unmap_work_handler); return dqm; } @@ -3057,6 +3098,7 @@ void device_queue_manager_uninit(struct device_queue_manager *dqm) { + cancel_delayed_work_sync(&dqm->notify_unmap_work); dqm->ops.stop(dqm); dqm->ops.uninitialize(dqm); if (!dqm->dev->kfd->shared_resources.enable_mes) --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h @@ -280,6 +280,7 @@ uint32_t wait_times; wait_queue_head_t destroy_wait; + struct delayed_work notify_unmap_work; /* for per-queue reset support */ struct dqm_detect_hang_info *detect_hang_info;