p71924506
/
linux_old1
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge tag 'amd-drm-next-5.19-2022-04-15' of https://gitlab.freedesktop.org/agd5f/linux into drm-next 

amd-drm-next-5.19-2022-04-15:

amdgpu:
- USB-C updates
- GPUVM updates
- TMZ fixes for RV
- DCN 3.1 pstate fixes
- Display z state fixes
- RAS fixes
- Misc code cleanups and spelling fixes
- More DC FP rework
- GPUVM TLB handling rework
- Power management sysfs code cleanup
- Add RAS support for VCN
- Backlight fix
- Add unique id support for more asics
- Misc display updates
- SR-IOV fixes
- Extend CG and PG flags to 64 bits
- Enable VCN clk sysfs nodes for navi12

amdkfd:
- Fix IO link cleanup during device removal
- RAS fixes
- Retry fault fixes
- Asynchronously free events
- SVM fixes

radeon:
- Drop some dead code
- Misc code cleanups

From: Alex Deucher <alexander.deucher@amd.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20220415135144.5700-1-alexander.deucher@amd.com
Signed-off-by: Dave Airlie <airlied@redhat.com>
This commit is contained in:
Dave Airlie 2022-04-28 14:33:20 +10:00
parent 19df0cfa25 d68cf992de
commit dbe946287e
184 changed files with 4359 additions and 4069 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
drivers/gpu/drm/amd/amdgpu/Makefile
View File

@ -49,7 +49,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
amdgpu_cs.o amdgpu_bios.o amdgpu_benchmark.o \
atombios_dp.o amdgpu_afmt.o amdgpu_trace_points.o \
atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
amdgpu_dma_buf.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
amdgpu_dma_buf.o amdgpu_vm.o amdgpu_vm_pt.o amdgpu_ib.o amdgpu_pll.o \
amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
amdgpu_gtt_mgr.o amdgpu_preempt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o \
amdgpu_atomfirmware.o amdgpu_vf_error.o amdgpu_sched.o \

6
drivers/gpu/drm/amd/amdgpu/amdgpu.h
View File

@ -179,7 +179,7 @@ extern int amdgpu_sched_jobs;
extern int amdgpu_sched_hw_submission;
extern uint amdgpu_pcie_gen_cap;
extern uint amdgpu_pcie_lane_cap;
extern uint amdgpu_cg_mask;
extern u64 amdgpu_cg_mask;
extern uint amdgpu_pg_mask;
extern uint amdgpu_sdma_phase_quantum;
extern char *amdgpu_disable_cu;
@ -322,7 +322,7 @@ int amdgpu_device_ip_set_powergating_state(void *dev,
enum amd_ip_block_type block_type,
enum amd_powergating_state state);
void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
u32 *flags);
u64 *flags);
int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
enum amd_ip_block_type block_type);
bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
@ -860,7 +860,7 @@ struct amdgpu_device {
/* powerplay */
struct amd_powerplay powerplay;
struct amdgpu_pm pm;
u32 cg_flags;
u64 cg_flags;
u32 pg_flags;
/* nbio */

8
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
View File

@ -724,3 +724,11 @@ void amdgpu_amdkfd_ras_poison_consumption_handler(struct amdgpu_device *adev, bo
else if (reset)
amdgpu_amdkfd_gpu_reset(adev);
}
bool amdgpu_amdkfd_ras_query_utcl2_poison_status(struct amdgpu_device *adev)
{
if (adev->gfx.ras && adev->gfx.ras->query_utcl2_poison_status)
return adev->gfx.ras->query_utcl2_poison_status(adev);
else
return false;
}

6
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
View File

@ -273,9 +273,8 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv,
uint64_t *size);
int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv,
bool *table_freed);
int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(struct amdgpu_device *adev,
struct kgd_mem *mem, void *drm_priv);
int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv);
int amdgpu_amdkfd_gpuvm_sync_memory(
@ -301,6 +300,7 @@ void amdgpu_amdkfd_ras_poison_consumption_handler(struct amdgpu_device *adev,
bool amdgpu_amdkfd_bo_mapped_to_dev(struct amdgpu_device *adev, struct kgd_mem *mem);
void amdgpu_amdkfd_block_mmu_notifications(void *p);
int amdgpu_amdkfd_criu_resume(void *p);
bool amdgpu_amdkfd_ras_query_utcl2_poison_status(struct amdgpu_device *adev);
#if IS_ENABLED(CONFIG_HSA_AMD)
void amdgpu_amdkfd_gpuvm_init_mem_limits(void);

18
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
View File

@ -1058,8 +1058,7 @@ static void unmap_bo_from_gpuvm(struct kgd_mem *mem,
static int update_gpuvm_pte(struct kgd_mem *mem,
struct kfd_mem_attachment *entry,
struct amdgpu_sync *sync,
bool *table_freed)
struct amdgpu_sync *sync)
{
struct amdgpu_bo_va *bo_va = entry->bo_va;
struct amdgpu_device *adev = entry->adev;
@ -1070,7 +1069,7 @@ static int update_gpuvm_pte(struct kgd_mem *mem,
return ret;
/* Update the page tables */
ret = amdgpu_vm_bo_update(adev, bo_va, false, table_freed);
ret = amdgpu_vm_bo_update(adev, bo_va, false);
if (ret) {
pr_err("amdgpu_vm_bo_update failed\n");
return ret;
@ -1082,8 +1081,7 @@ static int update_gpuvm_pte(struct kgd_mem *mem,
static int map_bo_to_gpuvm(struct kgd_mem *mem,
struct kfd_mem_attachment *entry,
struct amdgpu_sync *sync,
bool no_update_pte,
bool *table_freed)
bool no_update_pte)
{
int ret;
@ -1100,7 +1098,7 @@ static int map_bo_to_gpuvm(struct kgd_mem *mem,
if (no_update_pte)
return 0;
ret = update_gpuvm_pte(mem, entry, sync, table_freed);
ret = update_gpuvm_pte(mem, entry, sync);
if (ret) {
pr_err("update_gpuvm_pte() failed\n");
goto update_gpuvm_pte_failed;
@ -1710,7 +1708,7 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
struct amdgpu_device *adev, struct kgd_mem *mem,
void *drm_priv, bool *table_freed)
void *drm_priv)
{
struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
int ret;
@ -1797,7 +1795,7 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
entry->va, entry->va + bo_size, entry);
ret = map_bo_to_gpuvm(mem, entry, ctx.sync,
is_invalid_userptr, table_freed);
is_invalid_userptr);
if (ret) {
pr_err("Failed to map bo to gpuvm\n");
goto out_unreserve;
@ -2265,7 +2263,7 @@ static int validate_invalid_user_pages(struct amdkfd_process_info *process_info)
continue;
kfd_mem_dmaunmap_attachment(mem, attachment);
ret = update_gpuvm_pte(mem, attachment, &sync, NULL);
ret = update_gpuvm_pte(mem, attachment, &sync);
if (ret) {
pr_err("%s: update PTE failed\n", __func__);
/* make sure this gets validated again */
@ -2476,7 +2474,7 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence **ef)
continue;
kfd_mem_dmaunmap_attachment(mem, attachment);
ret = update_gpuvm_pte(mem, attachment, &sync_obj, NULL);
ret = update_gpuvm_pte(mem, attachment, &sync_obj);
if (ret) {
pr_debug("Memory eviction: update PTE failed. Try again\n");
goto validate_map_fail;

14
drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
View File

@ -785,22 +785,22 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
if (r)
return r;
r = amdgpu_vm_bo_update(adev, fpriv->prt_va, false, NULL);
r = amdgpu_vm_bo_update(adev, fpriv->prt_va, false);
if (r)
return r;
r = amdgpu_sync_vm_fence(&p->job->sync, fpriv->prt_va->last_pt_update);
r = amdgpu_sync_fence(&p->job->sync, fpriv->prt_va->last_pt_update);
if (r)
return r;
if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) {
bo_va = fpriv->csa_va;
BUG_ON(!bo_va);
r = amdgpu_vm_bo_update(adev, bo_va, false, NULL);
r = amdgpu_vm_bo_update(adev, bo_va, false);
if (r)
return r;
r = amdgpu_sync_vm_fence(&p->job->sync, bo_va->last_pt_update);
r = amdgpu_sync_fence(&p->job->sync, bo_va->last_pt_update);
if (r)
return r;
}
@ -815,11 +815,11 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
if (bo_va == NULL)
continue;
r = amdgpu_vm_bo_update(adev, bo_va, false, NULL);
r = amdgpu_vm_bo_update(adev, bo_va, false);
if (r)
return r;
r = amdgpu_sync_vm_fence(&p->job->sync, bo_va->last_pt_update);
r = amdgpu_sync_fence(&p->job->sync, bo_va->last_pt_update);
if (r)
return r;
}
@ -832,7 +832,7 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
if (r)
return r;
r = amdgpu_sync_vm_fence(&p->job->sync, vm->last_update);
r = amdgpu_sync_fence(&p->job->sync, vm->last_update);
if (r)
return r;

10
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
View File

@ -730,7 +730,7 @@ static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
return -ENOMEM;
/* version, increment each time something is added */
config[no_regs++] = 4;
config[no_regs++] = 5;
config[no_regs++] = adev->gfx.config.max_shader_engines;
config[no_regs++] = adev->gfx.config.max_tile_pipes;
config[no_regs++] = adev->gfx.config.max_cu_per_sh;
@ -757,8 +757,8 @@ static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
/* rev==1 */
config[no_regs++] = adev->rev_id;
config[no_regs++] = adev->pg_flags;
config[no_regs++] = adev->cg_flags;
config[no_regs++] = lower_32_bits(adev->pg_flags);
config[no_regs++] = lower_32_bits(adev->cg_flags);
/* rev==2 */
config[no_regs++] = adev->family;
@ -773,6 +773,10 @@ static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
/* rev==4 APU flag */
config[no_regs++] = adev->flags & AMD_IS_APU ? 1 : 0;
/* rev==5 PG/CG flag upper 32bit */
config[no_regs++] = upper_32_bits(adev->pg_flags);
config[no_regs++] = upper_32_bits(adev->cg_flags);
while (size && (*pos < no_regs * 4)) {
uint32_t value;

2
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
View File

@ -1703,7 +1703,7 @@ int amdgpu_device_ip_set_powergating_state(void *dev,
* clockgating is enabled.
*/
void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
u32 *flags)
u64 *flags)
{
int i;

2
drivers/gpu/drm/amd/amdgpu/amdgpu_df.h
View File

@ -40,7 +40,7 @@ struct amdgpu_df_funcs {
void (*update_medium_grain_clock_gating)(struct amdgpu_device *adev,
bool enable);
void (*get_clockgating_state)(struct amdgpu_device *adev,
u32 *flags);
u64 *flags);
void (*enable_ecc_force_par_wr_rmw)(struct amdgpu_device *adev,
bool enable);
int (*pmc_start)(struct amdgpu_device *adev, uint64_t config,

15
drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
View File

@ -430,7 +430,7 @@ static void amdgpu_discovery_read_harvest_bit_per_ip(struct amdgpu_device *adev,
}
}
next_ip:
ip_offset += sizeof(*ip) + 4 * (ip->num_base_address - 1);
ip_offset += struct_size(ip, base_address, ip->num_base_address);
}
}
}
@ -798,7 +798,7 @@ static int amdgpu_discovery_sysfs_ips(struct amdgpu_device *adev,
res = kobject_add(&ip_hw_instance->kobj, NULL,
"%d", ip_hw_instance->num_instance);
next_ip:
ip_offset += sizeof(*ip) + 4 * (ip->num_base_address - 1);
ip_offset += struct_size(ip, base_address, ip->num_base_address);
}
}
@ -1063,7 +1063,7 @@ int amdgpu_discovery_reg_base_init(struct amdgpu_device *adev)
}
next_ip:
ip_offset += sizeof(*ip) + 4 * (ip->num_base_address - 1);
ip_offset += struct_size(ip, base_address, ip->num_base_address);
}
}
@ -1113,7 +1113,7 @@ int amdgpu_discovery_get_ip_version(struct amdgpu_device *adev, int hw_id, int n
*revision = ip->revision;
return 0;
}
ip_offset += sizeof(*ip) + 4 * (ip->num_base_address - 1);
ip_offset += struct_size(ip, base_address, ip->num_base_address);
}
}
@ -1150,13 +1150,6 @@ void amdgpu_discovery_harvest_ip(struct amdgpu_device *adev)
adev->harvest_ip_mask |= AMD_HARVEST_IP_VCN_MASK;
adev->harvest_ip_mask |= AMD_HARVEST_IP_JPEG_MASK;
}
if ((adev->pdev->device == 0x731E &&
(adev->pdev->revision == 0xC6 || adev->pdev->revision == 0xC7)) ||
(adev->pdev->device == 0x7340 && adev->pdev->revision == 0xC9) ||
(adev->pdev->device == 0x7360 && adev->pdev->revision == 0xC7)) {
adev->harvest_ip_mask |= AMD_HARVEST_IP_VCN_MASK;
adev->harvest_ip_mask |= AMD_HARVEST_IP_JPEG_MASK;
}
}
union gc_info {

52
drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
View File

@ -41,6 +41,11 @@
#include <drm/drm_fourcc.h>
#include <drm/drm_vblank.h>
static int amdgpu_display_framebuffer_init(struct drm_device *dev,
struct amdgpu_framebuffer *rfb,
const struct drm_mode_fb_cmd2 *mode_cmd,
struct drm_gem_object *obj);
static void amdgpu_display_flip_callback(struct dma_fence *f,
struct dma_fence_cb *cb)
{
@ -113,8 +118,9 @@ static void amdgpu_display_flip_work_func(struct work_struct *__work)
spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
DRM_DEBUG_DRIVER("crtc:%d[%p], pflip_stat:AMDGPU_FLIP_SUBMITTED, work: %p,\n",
amdgpu_crtc->crtc_id, amdgpu_crtc, work);
drm_dbg_vbl(adev_to_drm(adev),
"crtc:%d[%p], pflip_stat:AMDGPU_FLIP_SUBMITTED, work: %p,\n",
amdgpu_crtc->crtc_id, amdgpu_crtc, work);
}
@ -1038,35 +1044,11 @@ static int amdgpu_display_get_fb_info(const struct amdgpu_framebuffer *amdgpu_fb
return r;
}
int amdgpu_display_gem_fb_init(struct drm_device *dev,
struct amdgpu_framebuffer *rfb,
const struct drm_mode_fb_cmd2 *mode_cmd,
struct drm_gem_object *obj)
{
int ret;
rfb->base.obj[0] = obj;
drm_helper_mode_fill_fb_struct(dev, &rfb->base, mode_cmd);
ret = amdgpu_display_framebuffer_init(dev, rfb, mode_cmd, obj);
if (ret)
goto err;
ret = drm_framebuffer_init(dev, &rfb->base, &amdgpu_fb_funcs);
if (ret)
goto err;
return 0;
err:
drm_dbg_kms(dev, "Failed to init gem fb: %d\n", ret);
rfb->base.obj[0] = NULL;
return ret;
}
int amdgpu_display_gem_fb_verify_and_init(
struct drm_device *dev, struct amdgpu_framebuffer *rfb,
struct drm_file *file_priv, const struct drm_mode_fb_cmd2 *mode_cmd,
struct drm_gem_object *obj)
static int amdgpu_display_gem_fb_verify_and_init(struct drm_device *dev,
struct amdgpu_framebuffer *rfb,
struct drm_file *file_priv,
const struct drm_mode_fb_cmd2 *mode_cmd,
struct drm_gem_object *obj)
{
int ret;
@ -1098,10 +1080,10 @@ int amdgpu_display_gem_fb_verify_and_init(
return ret;
}
int amdgpu_display_framebuffer_init(struct drm_device *dev,
struct amdgpu_framebuffer *rfb,
const struct drm_mode_fb_cmd2 *mode_cmd,
struct drm_gem_object *obj)
static int amdgpu_display_framebuffer_init(struct drm_device *dev,
struct amdgpu_framebuffer *rfb,
const struct drm_mode_fb_cmd2 *mode_cmd,
struct drm_gem_object *obj)
{
struct amdgpu_device *adev = drm_to_adev(dev);
int ret, i;

26
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
View File

@ -136,7 +136,7 @@ int amdgpu_sched_jobs = 32;
int amdgpu_sched_hw_submission = 2;
uint amdgpu_pcie_gen_cap;
uint amdgpu_pcie_lane_cap;
uint amdgpu_cg_mask = 0xffffffff;
u64 amdgpu_cg_mask = 0xffffffffffffffff;
uint amdgpu_pg_mask = 0xffffffff;
uint amdgpu_sdma_phase_quantum = 32;
char *amdgpu_disable_cu = NULL;
@ -454,12 +454,12 @@ MODULE_PARM_DESC(pcie_lane_cap, "PCIE Lane Caps (0: autodetect (default))");
module_param_named(pcie_lane_cap, amdgpu_pcie_lane_cap, uint, 0444);
/**
* DOC: cg_mask (uint)
* DOC: cg_mask (ullong)
* Override Clockgating features enabled on GPU (0 = disable clock gating). See the AMD_CG_SUPPORT flags in
* drivers/gpu/drm/amd/include/amd_shared.h. The default is 0xffffffff (all enabled).
* drivers/gpu/drm/amd/include/amd_shared.h. The default is 0xffffffffffffffff (all enabled).
*/
MODULE_PARM_DESC(cg_mask, "Clockgating flags mask (0 = disable clock gating)");
module_param_named(cg_mask, amdgpu_cg_mask, uint, 0444);
module_param_named(cg_mask, amdgpu_cg_mask, ullong, 0444);
/**
* DOC: pg_mask (uint)
@ -2323,18 +2323,23 @@ static int amdgpu_pmops_suspend(struct device *dev)
{
struct drm_device *drm_dev = dev_get_drvdata(dev);
struct amdgpu_device *adev = drm_to_adev(drm_dev);
int r;
if (amdgpu_acpi_is_s0ix_active(adev))
adev->in_s0ix = true;
else
adev->in_s3 = true;
r = amdgpu_device_suspend(drm_dev, true);
if (r)
return r;
return amdgpu_device_suspend(drm_dev, true);
}
static int amdgpu_pmops_suspend_noirq(struct device *dev)
{
struct drm_device *drm_dev = dev_get_drvdata(dev);
struct amdgpu_device *adev = drm_to_adev(drm_dev);
if (!adev->in_s0ix)
r = amdgpu_asic_reset(adev);
return r;
return amdgpu_asic_reset(adev);
return 0;
}
static int amdgpu_pmops_resume(struct device *dev)
@ -2575,6 +2580,7 @@ static const struct dev_pm_ops amdgpu_pm_ops = {
.prepare = amdgpu_pmops_prepare,
.complete = amdgpu_pmops_complete,
.suspend = amdgpu_pmops_suspend,
.suspend_noirq = amdgpu_pmops_suspend_noirq,
.resume = amdgpu_pmops_resume,
.freeze = amdgpu_pmops_freeze,
.thaw = amdgpu_pmops_thaw,

2
drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
View File

@ -613,7 +613,7 @@ static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
if (operation == AMDGPU_VA_OP_MAP ||
operation == AMDGPU_VA_OP_REPLACE) {
r = amdgpu_vm_bo_update(adev, bo_va, false, NULL);
r = amdgpu_vm_bo_update(adev, bo_va, false);
if (r)
goto error;
}

1
drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
View File

@ -202,6 +202,7 @@ struct amdgpu_cu_info {
struct amdgpu_gfx_ras {
struct amdgpu_ras_block_object ras_block;
void (*enable_watchdog_timer)(struct amdgpu_device *adev);
bool (*query_utcl2_poison_status)(struct amdgpu_device *adev);
};
struct amdgpu_gfx_funcs {

13
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
View File

@ -25,6 +25,9 @@
*/
#include <linux/io-64-nonatomic-lo-hi.h>
#ifdef CONFIG_X86
#include <asm/hypervisor.h>
#endif
#include "amdgpu.h"
#include "amdgpu_gmc.h"
@ -647,12 +650,14 @@ void amdgpu_gmc_get_vbios_allocations(struct amdgpu_device *adev)
case CHIP_VEGA10:
adev->mman.keep_stolen_vga_memory = true;
/*
* VEGA10 SRIOV VF needs some firmware reserved area.
* VEGA10 SRIOV VF with MS_HYPERV host needs some firmware reserved area.
*/
if (amdgpu_sriov_vf(adev)) {
adev->mman.stolen_reserved_offset = 0x100000;
adev->mman.stolen_reserved_size = 0x600000;
#ifdef CONFIG_X86
if (amdgpu_sriov_vf(adev) && hypervisor_is_type(X86_HYPER_MS_HYPERV)) {
adev->mman.stolen_reserved_offset = 0x500000;
adev->mman.stolen_reserved_size = 0x200000;
}
#endif
break;
case CHIP_RAVEN:
case CHIP_RENOIR:

2
drivers/gpu/drm/amd/amdgpu/amdgpu_hdp.h
View File

@ -33,7 +33,7 @@ struct amdgpu_hdp_funcs {
void (*invalidate_hdp)(struct amdgpu_device *adev,
struct amdgpu_ring *ring);
void (*update_clock_gating)(struct amdgpu_device *adev, bool enable);
void (*get_clock_gating_state)(struct amdgpu_device *adev, u32 *flags);
void (*get_clock_gating_state)(struct amdgpu_device *adev, u64 *flags);
void (*init_registers)(struct amdgpu_device *adev);
};

40
drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c
View File

@ -260,19 +260,15 @@ static int amdgpu_vmid_grab_reserved(struct amdgpu_vm *vm,
struct amdgpu_device *adev = ring->adev;
unsigned vmhub = ring->funcs->vmhub;
uint64_t fence_context = adev->fence_context + ring->idx;
struct dma_fence *updates = sync->last_vm_update;
bool needs_flush = vm->use_cpu_for_update;
int r = 0;
uint64_t updates = amdgpu_vm_tlb_seq(vm);
int r;
*id = vm->reserved_vmid[vmhub];
if (updates && (*id)->flushed_updates &&
updates->context == (*id)->flushed_updates->context &&
!dma_fence_is_later(updates, (*id)->flushed_updates))
updates = NULL;
if ((*id)->owner != vm->immediate.fence_context ||
job->vm_pd_addr != (*id)->pd_gpu_addr ||
updates || !(*id)->last_flush ||
(*id)->pd_gpu_addr != job->vm_pd_addr ||
(*id)->flushed_updates < updates ||
!(*id)->last_flush ||
((*id)->last_flush->context != fence_context &&
!dma_fence_is_signaled((*id)->last_flush))) {
struct dma_fence *tmp;
@ -286,8 +282,7 @@ static int amdgpu_vmid_grab_reserved(struct amdgpu_vm *vm,
tmp = amdgpu_sync_peek_fence(&(*id)->active, ring);
if (tmp) {
*id = NULL;
r = amdgpu_sync_fence(sync, tmp);
return r;
return amdgpu_sync_fence(sync, tmp);
}
needs_flush = true;
}
@ -299,10 +294,7 @@ static int amdgpu_vmid_grab_reserved(struct amdgpu_vm *vm,
if (r)
return r;
if (updates) {
dma_fence_put((*id)->flushed_updates);
(*id)->flushed_updates = dma_fence_get(updates);
}
(*id)->flushed_updates = updates;
job->vm_needs_flush = needs_flush;
return 0;
}
@ -330,7 +322,7 @@ static int amdgpu_vmid_grab_used(struct amdgpu_vm *vm,
unsigned vmhub = ring->funcs->vmhub;
struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
uint64_t fence_context = adev->fence_context + ring->idx;
struct dma_fence *updates = sync->last_vm_update;
uint64_t updates = amdgpu_vm_tlb_seq(vm);
int r;
job->vm_needs_flush = vm->use_cpu_for_update;
@ -338,7 +330,6 @@ static int amdgpu_vmid_grab_used(struct amdgpu_vm *vm,
/* Check if we can use a VMID already assigned to this VM */
list_for_each_entry_reverse((*id), &id_mgr->ids_lru, list) {
bool needs_flush = vm->use_cpu_for_update;
struct dma_fence *flushed;
/* Check all the prerequisites to using this VMID */
if ((*id)->owner != vm->immediate.fence_context)
@ -352,8 +343,7 @@ static int amdgpu_vmid_grab_used(struct amdgpu_vm *vm,
!dma_fence_is_signaled((*id)->last_flush)))
needs_flush = true;
flushed = (*id)->flushed_updates;
if (updates && (!flushed || dma_fence_is_later(updates, flushed)))
if ((*id)->flushed_updates < updates)
needs_flush = true;
if (needs_flush && !adev->vm_manager.concurrent_flush)
@ -366,11 +356,7 @@ static int amdgpu_vmid_grab_used(struct amdgpu_vm *vm,
if (r)
return r;
if (updates && (!flushed || dma_fence_is_later(updates, flushed))) {
dma_fence_put((*id)->flushed_updates);
(*id)->flushed_updates = dma_fence_get(updates);
}
(*id)->flushed_updates = updates;
job->vm_needs_flush |= needs_flush;
return 0;
}
@ -416,8 +402,6 @@ int amdgpu_vmid_grab(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
goto error;
if (!id) {
struct dma_fence *updates = sync->last_vm_update;
/* Still no ID to use? Then use the idle one found earlier */
id = idle;
@ -426,8 +410,7 @@ int amdgpu_vmid_grab(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
if (r)
goto error;
dma_fence_put(id->flushed_updates);
id->flushed_updates = dma_fence_get(updates);
id->flushed_updates = amdgpu_vm_tlb_seq(vm);
job->vm_needs_flush = true;
}
@ -594,7 +577,6 @@ void amdgpu_vmid_mgr_fini(struct amdgpu_device *adev)
struct amdgpu_vmid *id = &id_mgr->ids[j];
amdgpu_sync_free(&id->active);
dma_fence_put(id->flushed_updates);
dma_fence_put(id->last_flush);
dma_fence_put(id->pasid_mapping);
}

2
drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h
View File

@ -47,7 +47,7 @@ struct amdgpu_vmid {
uint64_t pd_gpu_addr;
/* last flushed PD/PT update */
struct dma_fence *flushed_updates;
uint64_t flushed_updates;
uint32_t current_gpu_reset_count;

8
drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.h
View File

@ -24,6 +24,8 @@
#ifndef __AMDGPU_JPEG_H__
#define __AMDGPU_JPEG_H__
#include "amdgpu_ras.h"
#define AMDGPU_MAX_JPEG_INSTANCES 2
#define AMDGPU_JPEG_HARVEST_JPEG0 (1 << 0)
@ -39,6 +41,10 @@ struct amdgpu_jpeg_inst {
struct amdgpu_jpeg_reg external;
};
struct amdgpu_jpeg_ras {
struct amdgpu_ras_block_object ras_block;
};
struct amdgpu_jpeg {
uint8_t num_jpeg_inst;
struct amdgpu_jpeg_inst inst[AMDGPU_MAX_JPEG_INSTANCES];
@ -48,6 +54,8 @@ struct amdgpu_jpeg {
enum amd_powergating_state cur_state;
struct mutex jpeg_pg_lock;
atomic_t total_submission_cnt;
struct ras_common_if *ras_if;
struct amdgpu_jpeg_ras *ras;
};
int amdgpu_jpeg_sw_init(struct amdgpu_device *adev);

2
drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h
View File

@ -34,7 +34,7 @@ struct amdgpu_mmhub_funcs {
void (*gart_disable)(struct amdgpu_device *adev);
int (*set_clockgating)(struct amdgpu_device *adev,
enum amd_clockgating_state state);
void (*get_clockgating)(struct amdgpu_device *adev, u32 *flags);
void (*get_clockgating)(struct amdgpu_device *adev, u64 *flags);
void (*setup_vm_pt_regs)(struct amdgpu_device *adev, uint32_t vmid,
uint64_t page_table_base);
void (*update_power_gating)(struct amdgpu_device *adev,

13
drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
View File

@ -592,19 +592,6 @@ int amdgpu_display_get_crtc_scanoutpos(struct drm_device *dev,
int *hpos, ktime_t *stime, ktime_t *etime,
const struct drm_display_mode *mode);
int amdgpu_display_gem_fb_init(struct drm_device *dev,
struct amdgpu_framebuffer *rfb,
const struct drm_mode_fb_cmd2 *mode_cmd,
struct drm_gem_object *obj);
int amdgpu_display_gem_fb_verify_and_init(
struct drm_device *dev, struct amdgpu_framebuffer *rfb,
struct drm_file *file_priv, const struct drm_mode_fb_cmd2 *mode_cmd,
struct drm_gem_object *obj);
int amdgpu_display_framebuffer_init(struct drm_device *dev,
struct amdgpu_framebuffer *rfb,
const struct drm_mode_fb_cmd2 *mode_cmd,
struct drm_gem_object *obj);
int amdgpufb_remove(struct drm_device *dev, struct drm_framebuffer *fb);
void amdgpu_enc_destroy(struct drm_encoder *encoder);

2
drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h
View File

@ -83,7 +83,7 @@ struct amdgpu_nbio_funcs {
void (*update_medium_grain_light_sleep)(struct amdgpu_device *adev,
bool enable);
void (*get_clockgating_state)(struct amdgpu_device *adev,
u32 *flags);
u64 *flags);
void (*ih_control)(struct amdgpu_device *adev);
void (*init_registers)(struct amdgpu_device *adev);
void (*remap_hdp_registers)(struct amdgpu_device *adev);

2
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
View File

@ -472,7 +472,7 @@ static bool amdgpu_bo_validate_size(struct amdgpu_device *adev,
fail:
DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size,
man->size << PAGE_SHIFT);
man->size);
return false;
}

9
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
View File

@ -66,6 +66,8 @@ const char *ras_block_string[] = {
"mp1",
"fuse",
"mca",
"vcn",
"jpeg",
};
const char *ras_mca_block_string[] = {
@ -2205,6 +2207,13 @@ static void amdgpu_ras_check_supported(struct amdgpu_device *adev)
dev_info(adev->dev, "SRAM ECC is active.\n");
adev->ras_hw_enabled |= ~(1 << AMDGPU_RAS_BLOCK__UMC |
1 << AMDGPU_RAS_BLOCK__DF);
if (adev->ip_versions[VCN_HWIP][0] == IP_VERSION(2, 6, 0))
adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__VCN |
1 << AMDGPU_RAS_BLOCK__JPEG);
else
adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__VCN |
1 << AMDGPU_RAS_BLOCK__JPEG);
} else {
dev_info(adev->dev, "SRAM ECC is not presented.\n");
}

3
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h
View File

@ -49,6 +49,8 @@ enum amdgpu_ras_block {
AMDGPU_RAS_BLOCK__MP1,
AMDGPU_RAS_BLOCK__FUSE,
AMDGPU_RAS_BLOCK__MCA,
AMDGPU_RAS_BLOCK__VCN,
AMDGPU_RAS_BLOCK__JPEG,
AMDGPU_RAS_BLOCK__LAST
};
@ -506,6 +508,7 @@ struct amdgpu_ras_block_hw_ops {
void (*query_ras_error_address)(struct amdgpu_device *adev, void *ras_error_status);
void (*reset_ras_error_count)(struct amdgpu_device *adev);
void (*reset_ras_error_status)(struct amdgpu_device *adev);
bool (*query_poison_status)(struct amdgpu_device *adev);
};
/* work flow

2
drivers/gpu/drm/amd/amdgpu/amdgpu_smuio.h
View File

@ -27,7 +27,7 @@ struct amdgpu_smuio_funcs {
u32 (*get_rom_index_offset)(struct amdgpu_device *adev);
u32 (*get_rom_data_offset)(struct amdgpu_device *adev);
void (*update_rom_clock_gating)(struct amdgpu_device *adev, bool enable);
void (*get_clock_gating_state)(struct amdgpu_device *adev, u32 *flags);
void (*get_clock_gating_state)(struct amdgpu_device *adev, u64 *flags);
u32 (*get_die_id)(struct amdgpu_device *adev);
u32 (*get_socket_id)(struct amdgpu_device *adev);
bool (*is_host_gpu_xgmi_supported)(struct amdgpu_device *adev);

23
drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
View File

@ -51,7 +51,6 @@ static struct kmem_cache *amdgpu_sync_slab;
void amdgpu_sync_create(struct amdgpu_sync *sync)
{
hash_init(sync->fences);
sync->last_vm_update = NULL;
}
/**
@ -171,23 +170,6 @@ int amdgpu_sync_fence(struct amdgpu_sync *sync, struct dma_fence *f)
return 0;
}
/**
* amdgpu_sync_vm_fence - remember to sync to this VM fence
*
* @sync: sync object to add fence to
* @fence: the VM fence to add
*
* Add the fence to the sync object and remember it as VM update.
*/
int amdgpu_sync_vm_fence(struct amdgpu_sync *sync, struct dma_fence *fence)
{
if (!fence)
return 0;
amdgpu_sync_keep_later(&sync->last_vm_update, fence);
return amdgpu_sync_fence(sync, fence);
}
/* Determine based on the owner and mode if we should sync to a fence or not */
static bool amdgpu_sync_test_fence(struct amdgpu_device *adev,
enum amdgpu_sync_mode mode,
@ -377,9 +359,6 @@ int amdgpu_sync_clone(struct amdgpu_sync *source, struct amdgpu_sync *clone)
}
}
dma_fence_put(clone->last_vm_update);
clone->last_vm_update = dma_fence_get(source->last_vm_update);
return 0;
}
@ -420,8 +399,6 @@ void amdgpu_sync_free(struct amdgpu_sync *sync)
dma_fence_put(e->fence);
kmem_cache_free(amdgpu_sync_slab, e);
}
dma_fence_put(sync->last_vm_update);
}
/**

2
drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h
View File

@ -43,12 +43,10 @@ enum amdgpu_sync_mode {
*/
struct amdgpu_sync {
DECLARE_HASHTABLE(fences, 4);
struct dma_fence *last_vm_update;
};
void amdgpu_sync_create(struct amdgpu_sync *sync);
int amdgpu_sync_fence(struct amdgpu_sync *sync, struct dma_fence *f);
int amdgpu_sync_vm_fence(struct amdgpu_sync *sync, struct dma_fence *fence);
int amdgpu_sync_resv(struct amdgpu_device *adev, struct amdgpu_sync *sync,
struct dma_resv *resv, enum amdgpu_sync_mode mode,
void *owner);

2
drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
View File

@ -52,7 +52,7 @@
#define FIRMWARE_ALDEBARAN "amdgpu/aldebaran_vcn.bin"
#define FIRMWARE_BEIGE_GOBY "amdgpu/beige_goby_vcn.bin"
#define FIRMWARE_YELLOW_CARP "amdgpu/yellow_carp_vcn.bin"
#define FIRMWARE_VCN_3_1_2 "amdgpu/vcn_3_1_2_vcn.bin"
#define FIRMWARE_VCN_3_1_2 "amdgpu/vcn_3_1_2.bin"
MODULE_FIRMWARE(FIRMWARE_RAVEN);
MODULE_FIRMWARE(FIRMWARE_PICASSO);

9
drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h
View File

@ -24,6 +24,8 @@
#ifndef __AMDGPU_VCN_H__
#define __AMDGPU_VCN_H__
#include "amdgpu_ras.h"
#define AMDGPU_VCN_STACK_SIZE (128*1024)
#define AMDGPU_VCN_CONTEXT_SIZE (512*1024)
@ -233,6 +235,10 @@ struct amdgpu_vcn_inst {
struct amdgpu_vcn_fw_shared fw_shared;
};
struct amdgpu_vcn_ras {
struct amdgpu_ras_block_object ras_block;
};
struct amdgpu_vcn {
unsigned fw_version;
struct delayed_work idle_work;
@ -252,6 +258,9 @@ struct amdgpu_vcn {
unsigned harvest_config;
int (*pause_dpg_mode)(struct amdgpu_device *adev,
int inst_idx, struct dpg_pause_state *new_state);
struct ras_common_if *ras_if;
struct amdgpu_vcn_ras *ras;
};
struct amdgpu_fw_shared_rb_ptrs_struct {

24
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
View File

@ -23,6 +23,10 @@
#include <linux/module.h>
#ifdef CONFIG_X86
#include <asm/hypervisor.h>
#endif
#include <drm/drm_drv.h>
#include "amdgpu.h"
@ -723,8 +727,12 @@ void amdgpu_detect_virtualization(struct amdgpu_device *adev)
break;
case CHIP_VEGA10:
soc15_set_virt_ops(adev);
/* send a dummy GPU_INIT_DATA request to host on vega10 */
amdgpu_virt_request_init_data(adev);
#ifdef CONFIG_X86
/* not send GPU_INIT_DATA with MS_HYPERV*/
if (!hypervisor_is_type(X86_HYPER_MS_HYPERV))
#endif
/* send a dummy GPU_INIT_DATA request to host on vega10 */
amdgpu_virt_request_init_data(adev);
break;
case CHIP_VEGA20:
case CHIP_ARCTURUS:
@ -862,11 +870,11 @@ static u32 amdgpu_virt_rlcg_reg_rw(struct amdgpu_device *adev, u32 offset, u32 v
uint32_t timeout = 50000;
uint32_t i, tmp;
uint32_t ret = 0;
static void *scratch_reg0;
static void *scratch_reg1;
static void *scratch_reg2;
static void *scratch_reg3;
static void *spare_int;
void *scratch_reg0;
void *scratch_reg1;
void *scratch_reg2;
void *scratch_reg3;
void *spare_int;
if (!adev->gfx.rlc.rlcg_reg_access_supported) {
dev_err(adev->dev,
@ -919,7 +927,7 @@ static u32 amdgpu_virt_rlcg_reg_rw(struct amdgpu_device *adev, u32 offset, u32 v
"wrong operation type, rlcg failed to program reg: 0x%05x\n", offset);
} else if (tmp & AMDGPU_RLCG_REG_NOT_IN_RANGE) {
dev_err(adev->dev,
"regiser is not in range, rlcg failed to program reg: 0x%05x\n", offset);
"register is not in range, rlcg failed to program reg: 0x%05x\n", offset);
} else {
dev_err(adev->dev,
"unknown error type, rlcg failed to program reg: 0x%05x\n", offset);

1134
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
View File

File diff suppressed because it is too large Load Diff

49
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
View File

@ -284,6 +284,10 @@ struct amdgpu_vm {
struct drm_sched_entity immediate;
struct drm_sched_entity delayed;
/* Last finished delayed update */
atomic64_t tlb_seq;
struct dma_fence *last_tlb_flush;
/* Last unlocked submission to the scheduler entities */
struct dma_fence *last_unlocked;
@ -395,18 +399,17 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
struct dma_fence **fence);
int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
struct amdgpu_vm *vm);
int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
struct amdgpu_device *bo_adev,
struct amdgpu_vm *vm, bool immediate,
bool unlocked, struct dma_resv *resv,
uint64_t start, uint64_t last,
uint64_t flags, uint64_t offset,
struct ttm_resource *res,
dma_addr_t *pages_addr,
struct dma_fence **fence, bool *free_table);
void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
struct amdgpu_vm *vm, struct amdgpu_bo *bo);
int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
bool immediate, bool unlocked, bool flush_tlb,
struct dma_resv *resv, uint64_t start, uint64_t last,
uint64_t flags, uint64_t offset, uint64_t vram_base,
struct ttm_resource *res, dma_addr_t *pages_addr,
struct dma_fence **fence);
int amdgpu_vm_bo_update(struct amdgpu_device *adev,
struct amdgpu_bo_va *bo_va,
bool clear, bool *table_freed);
bool clear);
bool amdgpu_vm_evictable(struct amdgpu_bo *bo);
void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
struct amdgpu_bo *bo, bool evicted);
@ -455,8 +458,34 @@ void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev,
void amdgpu_vm_get_memory(struct amdgpu_vm *vm, uint64_t *vram_mem,
uint64_t *gtt_mem, uint64_t *cpu_mem);
int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
struct amdgpu_bo_vm *vmbo, bool immediate);
int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
int level, bool immediate, struct amdgpu_bo_vm **vmbo);
void amdgpu_vm_pt_free_root(struct amdgpu_device *adev, struct amdgpu_vm *vm);
bool amdgpu_vm_pt_is_root_clean(struct amdgpu_device *adev,
struct amdgpu_vm *vm);
int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params,
struct amdgpu_vm_bo_base *entry);
int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
uint64_t start, uint64_t end,
uint64_t dst, uint64_t flags);
#if defined(CONFIG_DEBUG_FS)
void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m);
#endif
/**
* amdgpu_vm_tlb_seq - return tlb flush sequence number
* @vm: the amdgpu_vm structure to query
*
* Returns the tlb flush sequence number which indicates that the VM TLBs needs
* to be invalidated whenever the sequence number change.
*/
static inline uint64_t amdgpu_vm_tlb_seq(struct amdgpu_vm *vm)
{
return atomic64_read(&vm->tlb_seq);
}
#endif

977
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c Normal file
View File

@ -0,0 +1,977 @@
// SPDX-License-Identifier: GPL-2.0 OR MIT
/*
* Copyright 2022 Advanced Micro Devices, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#include <drm/drm_drv.h>
#include "amdgpu.h"
#include "amdgpu_trace.h"
#include "amdgpu_vm.h"
/*
* amdgpu_vm_pt_cursor - state for for_each_amdgpu_vm_pt
*/
struct amdgpu_vm_pt_cursor {
uint64_t pfn;
struct amdgpu_vm_bo_base *parent;
struct amdgpu_vm_bo_base *entry;
unsigned int level;
};
/**
* amdgpu_vm_pt_level_shift - return the addr shift for each level
*
* @adev: amdgpu_device pointer
* @level: VMPT level
*
* Returns:
* The number of bits the pfn needs to be right shifted for a level.
*/
static unsigned int amdgpu_vm_pt_level_shift(struct amdgpu_device *adev,
unsigned int level)
{
switch (level) {
case AMDGPU_VM_PDB2:
case AMDGPU_VM_PDB1:
case AMDGPU_VM_PDB0:
return 9 * (AMDGPU_VM_PDB0 - level) +
adev->vm_manager.block_size;
case AMDGPU_VM_PTB:
return 0;
default:
return ~0;
}
}
/**
* amdgpu_vm_pt_num_entries - return the number of entries in a PD/PT
*
* @adev: amdgpu_device pointer
* @level: VMPT level
*
* Returns:
* The number of entries in a page directory or page table.
*/
static unsigned int amdgpu_vm_pt_num_entries(struct amdgpu_device *adev,
unsigned int level)
{
unsigned int shift;
shift = amdgpu_vm_pt_level_shift(adev, adev->vm_manager.root_level);
if (level == adev->vm_manager.root_level)
/* For the root directory */
return round_up(adev->vm_manager.max_pfn, 1ULL << shift)
>> shift;
else if (level != AMDGPU_VM_PTB)
/* Everything in between */
return 512;
/* For the page tables on the leaves */
return AMDGPU_VM_PTE_COUNT(adev);
}
/**
* amdgpu_vm_pt_num_ats_entries - return the number of ATS entries in the root PD
*
* @adev: amdgpu_device pointer
*
* Returns:
* The number of entries in the root page directory which needs the ATS setting.
*/
static unsigned int amdgpu_vm_pt_num_ats_entries(struct amdgpu_device *adev)
{
unsigned int shift;
shift = amdgpu_vm_pt_level_shift(adev, adev->vm_manager.root_level);
return AMDGPU_GMC_HOLE_START >> (shift + AMDGPU_GPU_PAGE_SHIFT);
}
/**
* amdgpu_vm_pt_entries_mask - the mask to get the entry number of a PD/PT
*
* @adev: amdgpu_device pointer
* @level: VMPT level
*
* Returns:
* The mask to extract the entry number of a PD/PT from an address.
*/
static uint32_t amdgpu_vm_pt_entries_mask(struct amdgpu_device *adev,
unsigned int level)
{
if (level <= adev->vm_manager.root_level)
return 0xffffffff;
else if (level != AMDGPU_VM_PTB)
return 0x1ff;
else
return AMDGPU_VM_PTE_COUNT(adev) - 1;
}
/**
* amdgpu_vm_pt_size - returns the size of the page table in bytes
*
* @adev: amdgpu_device pointer
* @level: VMPT level
*
* Returns:
* The size of the BO for a page directory or page table in bytes.
*/
static unsigned int amdgpu_vm_pt_size(struct amdgpu_device *adev,
unsigned int level)
{
return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_pt_num_entries(adev, level) * 8);
}
/**
* amdgpu_vm_pt_parent - get the parent page directory
*
* @pt: child page table
*
* Helper to get the parent entry for the child page table. NULL if we are at
* the root page directory.
*/
static struct amdgpu_vm_bo_base *
amdgpu_vm_pt_parent(struct amdgpu_vm_bo_base *pt)
{
struct amdgpu_bo *parent = pt->bo->parent;
if (!parent)
return NULL;
return parent->vm_bo;
}
/**
* amdgpu_vm_pt_start - start PD/PT walk
*
* @adev: amdgpu_device pointer
* @vm: amdgpu_vm structure
* @start: start address of the walk
* @cursor: state to initialize
*
* Initialize a amdgpu_vm_pt_cursor to start a walk.
*/
static void amdgpu_vm_pt_start(struct amdgpu_device *adev,
struct amdgpu_vm *vm, uint64_t start,
struct amdgpu_vm_pt_cursor *cursor)
{
cursor->pfn = start;
cursor->parent = NULL;
cursor->entry = &vm->root;
cursor->level = adev->vm_manager.root_level;
}
/**
* amdgpu_vm_pt_descendant - go to child node
*
* @adev: amdgpu_device pointer
* @cursor: current state
*
* Walk to the child node of the current node.
* Returns:
* True if the walk was possible, false otherwise.
*/
static bool amdgpu_vm_pt_descendant(struct amdgpu_device *adev,
struct amdgpu_vm_pt_cursor *cursor)
{
unsigned int mask, shift, idx;
if ((cursor->level == AMDGPU_VM_PTB) || !cursor->entry ||
!cursor->entry->bo)
return false;
mask = amdgpu_vm_pt_entries_mask(adev, cursor->level);
shift = amdgpu_vm_pt_level_shift(adev, cursor->level);
++cursor->level;
idx = (cursor->pfn >> shift) & mask;
cursor->parent = cursor->entry;
cursor->entry = &to_amdgpu_bo_vm(cursor->entry->bo)->entries[idx];
return true;
}
/**
* amdgpu_vm_pt_sibling - go to sibling node
*
* @adev: amdgpu_device pointer
* @cursor: current state
*
* Walk to the sibling node of the current node.
* Returns:
* True if the walk was possible, false otherwise.
*/
static bool amdgpu_vm_pt_sibling(struct amdgpu_device *adev,
struct amdgpu_vm_pt_cursor *cursor)
{
unsigned int shift, num_entries;
struct amdgpu_bo_vm *parent;
/* Root doesn't have a sibling */
if (!cursor->parent)
return false;
/* Go to our parents and see if we got a sibling */
shift = amdgpu_vm_pt_level_shift(adev, cursor->level - 1);
num_entries = amdgpu_vm_pt_num_entries(adev, cursor->level - 1);
parent = to_amdgpu_bo_vm(cursor->parent->bo);
if (cursor->entry == &parent->entries[num_entries - 1])
return false;
cursor->pfn += 1ULL << shift;
cursor->pfn &= ~((1ULL << shift) - 1);
++cursor->entry;
return true;
}
/**
* amdgpu_vm_pt_ancestor - go to parent node
*
* @cursor: current state
*
* Walk to the parent node of the current node.
* Returns:
* True if the walk was possible, false otherwise.
*/
static bool amdgpu_vm_pt_ancestor(struct amdgpu_vm_pt_cursor *cursor)
{
if (!cursor->parent)
return false;
--cursor->level;
cursor->entry = cursor->parent;
cursor->parent = amdgpu_vm_pt_parent(cursor->parent);
return true;
}
/**
* amdgpu_vm_pt_next - get next PD/PT in hieratchy
*
* @adev: amdgpu_device pointer
* @cursor: current state
*
* Walk the PD/PT tree to the next node.
*/
static void amdgpu_vm_pt_next(struct amdgpu_device *adev,
struct amdgpu_vm_pt_cursor *cursor)
{
/* First try a newborn child */
if (amdgpu_vm_pt_descendant(adev, cursor))
return;
/* If that didn't worked try to find a sibling */
while (!amdgpu_vm_pt_sibling(adev, cursor)) {
/* No sibling, go to our parents and grandparents */
if (!amdgpu_vm_pt_ancestor(cursor)) {
cursor->pfn = ~0ll;
return;
}
}
}
/**
* amdgpu_vm_pt_first_dfs - start a deep first search
*
* @adev: amdgpu_device structure
* @vm: amdgpu_vm structure
* @start: optional cursor to start with
* @cursor: state to initialize
*
* Starts a deep first traversal of the PD/PT tree.
*/
static void amdgpu_vm_pt_first_dfs(struct amdgpu_device *adev,
struct amdgpu_vm *vm,
struct amdgpu_vm_pt_cursor *start,
struct amdgpu_vm_pt_cursor *cursor)
{
if (start)
*cursor = *start;
else
amdgpu_vm_pt_start(adev, vm, 0, cursor);
while (amdgpu_vm_pt_descendant(adev, cursor))
;
}
/**
* amdgpu_vm_pt_continue_dfs - check if the deep first search should continue
*
* @start: starting point for the search
* @entry: current entry
*
* Returns:
* True when the search should continue, false otherwise.
*/
static bool amdgpu_vm_pt_continue_dfs(struct amdgpu_vm_pt_cursor *start,
struct amdgpu_vm_bo_base *entry)
{
return entry && (!start || entry != start->entry);
}
/**
* amdgpu_vm_pt_next_dfs - get the next node for a deep first search
*
* @adev: amdgpu_device structure
* @cursor: current state
*
* Move the cursor to the next node in a deep first search.
*/
static void amdgpu_vm_pt_next_dfs(struct amdgpu_device *adev,
struct amdgpu_vm_pt_cursor *cursor)
{
if (!cursor->entry)
return;
if (!cursor->parent)
cursor->entry = NULL;
else if (amdgpu_vm_pt_sibling(adev, cursor))
while (amdgpu_vm_pt_descendant(adev, cursor))
;
else
amdgpu_vm_pt_ancestor(cursor);
}
/*
* for_each_amdgpu_vm_pt_dfs_safe - safe deep first search of all PDs/PTs
*/
#define for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry) \
for (amdgpu_vm_pt_first_dfs((adev), (vm), (start), &(cursor)), \
(entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor));\
amdgpu_vm_pt_continue_dfs((start), (entry)); \
(entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor)))
/**
* amdgpu_vm_pt_clear - initially clear the PDs/PTs
*
* @adev: amdgpu_device pointer
* @vm: VM to clear BO from
* @vmbo: BO to clear
* @immediate: use an immediate update
*
* Root PD needs to be reserved when calling this.
*
* Returns:
* 0 on success, errno otherwise.
*/
int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
struct amdgpu_bo_vm *vmbo, bool immediate)
{
unsigned int level = adev->vm_manager.root_level;
struct ttm_operation_ctx ctx = { true, false };
struct amdgpu_vm_update_params params;
struct amdgpu_bo *ancestor = &vmbo->bo;
unsigned int entries, ats_entries;
struct amdgpu_bo *bo = &vmbo->bo;
uint64_t addr;
int r, idx;
/* Figure out our place in the hierarchy */
if (ancestor->parent) {
++level;
while (ancestor->parent->parent) {
++level;
ancestor = ancestor->parent;
}
}
entries = amdgpu_bo_size(bo) / 8;
if (!vm->pte_support_ats) {
ats_entries = 0;
} else if (!bo->parent) {
ats_entries = amdgpu_vm_pt_num_ats_entries(adev);
ats_entries = min(ats_entries, entries);
entries -= ats_entries;
} else {
struct amdgpu_vm_bo_base *pt;
pt = ancestor->vm_bo;
ats_entries = amdgpu_vm_pt_num_ats_entries(adev);
if ((pt - to_amdgpu_bo_vm(vm->root.bo)->entries) >=
ats_entries) {
ats_entries = 0;
} else {
ats_entries = entries;
entries = 0;
}
}
r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
if (r)
return r;
if (vmbo->shadow) {
struct amdgpu_bo *shadow = vmbo->shadow;
r = ttm_bo_validate(&shadow->tbo, &shadow->placement, &ctx);
if (r)
return r;
}
if (!drm_dev_enter(adev_to_drm(adev), &idx))
return -ENODEV;
r = vm->update_funcs->map_table(vmbo);
if (r)
goto exit;
memset(&params, 0, sizeof(params));
params.adev = adev;
params.vm = vm;
params.immediate = immediate;
r = vm->update_funcs->prepare(&params, NULL, AMDGPU_SYNC_EXPLICIT);
if (r)
goto exit;
addr = 0;
if (ats_entries) {
uint64_t value = 0, flags;
flags = AMDGPU_PTE_DEFAULT_ATC;
if (level != AMDGPU_VM_PTB) {
/* Handle leaf PDEs as PTEs */
flags |= AMDGPU_PDE_PTE;
amdgpu_gmc_get_vm_pde(adev, level, &value, &flags);
}
r = vm->update_funcs->update(&params, vmbo, addr, 0,
ats_entries, value, flags);
if (r)
goto exit;
addr += ats_entries * 8;
}
if (entries) {
uint64_t value = 0, flags = 0;
if (adev->asic_type >= CHIP_VEGA10) {
if (level != AMDGPU_VM_PTB) {
/* Handle leaf PDEs as PTEs */
flags |= AMDGPU_PDE_PTE;
amdgpu_gmc_get_vm_pde(adev, level,
&value, &flags);
} else {
/* Workaround for fault priority problem on GMC9 */
flags = AMDGPU_PTE_EXECUTABLE;
}
}
r = vm->update_funcs->update(&params, vmbo, addr, 0, entries,
value, flags);
if (r)
goto exit;
}
r = vm->update_funcs->commit(&params, NULL);
exit:
drm_dev_exit(idx);
return r;
}
/**
* amdgpu_vm_pt_create - create bo for PD/PT
*
* @adev: amdgpu_device pointer
* @vm: requesting vm
* @level: the page table level
* @immediate: use a immediate update
* @vmbo: pointer to the buffer object pointer
*/
int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
int level, bool immediate, struct amdgpu_bo_vm **vmbo)
{
struct amdgpu_bo_param bp;
struct amdgpu_bo *bo;
struct dma_resv *resv;
unsigned int num_entries;
int r;
memset(&bp, 0, sizeof(bp));
bp.size = amdgpu_vm_pt_size(adev, level);
bp.byte_align = AMDGPU_GPU_PAGE_SIZE;
bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
bp.domain = amdgpu_bo_get_preferred_domain(adev, bp.domain);
bp.flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
AMDGPU_GEM_CREATE_CPU_GTT_USWC;
if (level < AMDGPU_VM_PTB)
num_entries = amdgpu_vm_pt_num_entries(adev, level);
else
num_entries = 0;
bp.bo_ptr_size = struct_size((*vmbo), entries, num_entries);
if (vm->use_cpu_for_update)
bp.flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
bp.type = ttm_bo_type_kernel;
bp.no_wait_gpu = immediate;
if (vm->root.bo)
bp.resv = vm->root.bo->tbo.base.resv;
r = amdgpu_bo_create_vm(adev, &bp, vmbo);
if (r)
return r;
bo = &(*vmbo)->bo;
if (vm->is_compute_context || (adev->flags & AMD_IS_APU)) {
(*vmbo)->shadow = NULL;
return 0;
}
if (!bp.resv)
WARN_ON(dma_resv_lock(bo->tbo.base.resv,
NULL));
resv = bp.resv;
memset(&bp, 0, sizeof(bp));
bp.size = amdgpu_vm_pt_size(adev, level);
bp.domain = AMDGPU_GEM_DOMAIN_GTT;
bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC;
bp.type = ttm_bo_type_kernel;
bp.resv = bo->tbo.base.resv;
bp.bo_ptr_size = sizeof(struct amdgpu_bo);
r = amdgpu_bo_create(adev, &bp, &(*vmbo)->shadow);
if (!resv)
dma_resv_unlock(bo->tbo.base.resv);
if (r) {
amdgpu_bo_unref(&bo);
return r;
}
(*vmbo)->shadow->parent = amdgpu_bo_ref(bo);
amdgpu_bo_add_to_shadow_list(*vmbo);
return 0;
}
/**
* amdgpu_vm_pt_alloc - Allocate a specific page table
*
* @adev: amdgpu_device pointer
* @vm: VM to allocate page tables for
* @cursor: Which page table to allocate
* @immediate: use an immediate update
*
* Make sure a specific page table or directory is allocated.
*
* Returns:
* 1 if page table needed to be allocated, 0 if page table was already
* allocated, negative errno if an error occurred.
*/
static int amdgpu_vm_pt_alloc(struct amdgpu_device *adev,
struct amdgpu_vm *vm,
struct amdgpu_vm_pt_cursor *cursor,
bool immediate)
{
struct amdgpu_vm_bo_base *entry = cursor->entry;
struct amdgpu_bo *pt_bo;
struct amdgpu_bo_vm *pt;
int r;
if (entry->bo)
return 0;
r = amdgpu_vm_pt_create(adev, vm, cursor->level, immediate, &pt);
if (r)
return r;
/* Keep a reference to the root directory to avoid
* freeing them up in the wrong order.
*/
pt_bo = &pt->bo;
pt_bo->parent = amdgpu_bo_ref(cursor->parent->bo);
amdgpu_vm_bo_base_init(entry, vm, pt_bo);
r = amdgpu_vm_pt_clear(adev, vm, pt, immediate);
if (r)
goto error_free_pt;
return 0;
error_free_pt:
amdgpu_bo_unref(&pt->shadow);
amdgpu_bo_unref(&pt_bo);
return r;
}
/**
* amdgpu_vm_pt_free - free one PD/PT
*
* @entry: PDE to free
*/
static void amdgpu_vm_pt_free(struct amdgpu_vm_bo_base *entry)
{
struct amdgpu_bo *shadow;
if (!entry->bo)
return;
shadow = amdgpu_bo_shadowed(entry->bo);
entry->bo->vm_bo = NULL;
list_del(&entry->vm_status);
amdgpu_bo_unref(&shadow);
amdgpu_bo_unref(&entry->bo);
}
/**
* amdgpu_vm_pt_free_dfs - free PD/PT levels
*
* @adev: amdgpu device structure
* @vm: amdgpu vm structure
* @start: optional cursor where to start freeing PDs/PTs
*
* Free the page directory or page table level and all sub levels.
*/
static void amdgpu_vm_pt_free_dfs(struct amdgpu_device *adev,
struct amdgpu_vm *vm,
struct amdgpu_vm_pt_cursor *start)
{
struct amdgpu_vm_pt_cursor cursor;
struct amdgpu_vm_bo_base *entry;
for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry)
amdgpu_vm_pt_free(entry);
if (start)
amdgpu_vm_pt_free(start->entry);
}
/**
* amdgpu_vm_pt_free_root - free root PD
* @adev: amdgpu device structure
* @vm: amdgpu vm structure
*
* Free the root page directory and everything below it.
*/
void amdgpu_vm_pt_free_root(struct amdgpu_device *adev, struct amdgpu_vm *vm)
{
amdgpu_vm_pt_free_dfs(adev, vm, NULL);
}
/**
* amdgpu_vm_pt_is_root_clean - check if a root PD is clean
*
* @adev: amdgpu_device pointer
* @vm: the VM to check
*
* Check all entries of the root PD, if any subsequent PDs are allocated,
* it means there are page table creating and filling, and is no a clean
* VM
*
* Returns:
* 0 if this VM is clean
*/
bool amdgpu_vm_pt_is_root_clean(struct amdgpu_device *adev,
struct amdgpu_vm *vm)
{
enum amdgpu_vm_level root = adev->vm_manager.root_level;
unsigned int entries = amdgpu_vm_pt_num_entries(adev, root);
unsigned int i = 0;
for (i = 0; i < entries; i++) {
if (to_amdgpu_bo_vm(vm->root.bo)->entries[i].bo)
return false;
}
return true;
}
/**
* amdgpu_vm_pde_update - update a single level in the hierarchy
*
* @params: parameters for the update
* @entry: entry to update
*
* Makes sure the requested entry in parent is up to date.
*/
int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params,
struct amdgpu_vm_bo_base *entry)
{
struct amdgpu_vm_bo_base *parent = amdgpu_vm_pt_parent(entry);
struct amdgpu_bo *bo = parent->bo, *pbo;
struct amdgpu_vm *vm = params->vm;
uint64_t pde, pt, flags;
unsigned int level;
for (level = 0, pbo = bo->parent; pbo; ++level)
pbo = pbo->parent;
level += params->adev->vm_manager.root_level;
amdgpu_gmc_get_pde_for_bo(entry->bo, level, &pt, &flags);
pde = (entry - to_amdgpu_bo_vm(parent->bo)->entries) * 8;
return vm->update_funcs->update(params, to_amdgpu_bo_vm(bo), pde, pt,
1, 0, flags);
}
/*
* amdgpu_vm_pte_update_flags - figure out flags for PTE updates
*
* Make sure to set the right flags for the PTEs at the desired level.
*/
static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,
struct amdgpu_bo_vm *pt,
unsigned int level,
uint64_t pe, uint64_t addr,
unsigned int count, uint32_t incr,
uint64_t flags)
{
if (level != AMDGPU_VM_PTB) {
flags |= AMDGPU_PDE_PTE;
amdgpu_gmc_get_vm_pde(params->adev, level, &addr, &flags);
} else if (params->adev->asic_type >= CHIP_VEGA10 &&
!(flags & AMDGPU_PTE_VALID) &&
!(flags & AMDGPU_PTE_PRT)) {
/* Workaround for fault priority problem on GMC9 */
flags |= AMDGPU_PTE_EXECUTABLE;
}
params->vm->update_funcs->update(params, pt, pe, addr, count, incr,
flags);
}
/**
* amdgpu_vm_pte_fragment - get fragment for PTEs
*
* @params: see amdgpu_vm_update_params definition
* @start: first PTE to handle
* @end: last PTE to handle
* @flags: hw mapping flags
* @frag: resulting fragment size
* @frag_end: end of this fragment
*
* Returns the first possible fragment for the start and end address.
*/
static void amdgpu_vm_pte_fragment(struct amdgpu_vm_update_params *params,
uint64_t start, uint64_t end, uint64_t flags,
unsigned int *frag, uint64_t *frag_end)
{
/**
* The MC L1 TLB supports variable sized pages, based on a fragment
* field in the PTE. When this field is set to a non-zero value, page
* granularity is increased from 4KB to (1 << (12 + frag)). The PTE
* flags are considered valid for all PTEs within the fragment range
* and corresponding mappings are assumed to be physically contiguous.
*
* The L1 TLB can store a single PTE for the whole fragment,
* significantly increasing the space available for translation
* caching. This leads to large improvements in throughput when the
* TLB is under pressure.
*
* The L2 TLB distributes small and large fragments into two
* asymmetric partitions. The large fragment cache is significantly
* larger. Thus, we try to use large fragments wherever possible.
* Userspace can support this by aligning virtual base address and
* allocation size to the fragment size.
*
* Starting with Vega10 the fragment size only controls the L1. The L2
* is now directly feed with small/huge/giant pages from the walker.
*/
unsigned int max_frag;
if (params->adev->asic_type < CHIP_VEGA10)
max_frag = params->adev->vm_manager.fragment_size;
else
max_frag = 31;
/* system pages are non continuously */
if (params->pages_addr) {
*frag = 0;
*frag_end = end;
return;
}
/* This intentionally wraps around if no bit is set */
*frag = min_t(unsigned int, ffs(start) - 1, fls64(end - start) - 1);
if (*frag >= max_frag) {
*frag = max_frag;
*frag_end = end & ~((1ULL << max_frag) - 1);
} else {
*frag_end = start + (1 << *frag);
}
}
/**
* amdgpu_vm_ptes_update - make sure that page tables are valid
*
* @params: see amdgpu_vm_update_params definition
* @start: start of GPU address range
* @end: end of GPU address range
* @dst: destination address to map to, the next dst inside the function
* @flags: mapping flags
*
* Update the page tables in the range @start - @end.
*
* Returns:
* 0 for success, -EINVAL for failure.
*/
int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
uint64_t start, uint64_t end,
uint64_t dst, uint64_t flags)
{
struct amdgpu_device *adev = params->adev;
struct amdgpu_vm_pt_cursor cursor;
uint64_t frag_start = start, frag_end;
unsigned int frag;
int r;
/* figure out the initial fragment */
amdgpu_vm_pte_fragment(params, frag_start, end, flags, &frag,
&frag_end);
/* walk over the address space and update the PTs */
amdgpu_vm_pt_start(adev, params->vm, start, &cursor);
while (cursor.pfn < end) {
unsigned int shift, parent_shift, mask;
uint64_t incr, entry_end, pe_start;
struct amdgpu_bo *pt;
if (!params->unlocked) {
/* make sure that the page tables covering the
* address range are actually allocated
*/
r = amdgpu_vm_pt_alloc(params->adev, params->vm,
&cursor, params->immediate);
if (r)
return r;
}
shift = amdgpu_vm_pt_level_shift(adev, cursor.level);
parent_shift = amdgpu_vm_pt_level_shift(adev, cursor.level - 1);
if (params->unlocked) {
/* Unlocked updates are only allowed on the leaves */
if (amdgpu_vm_pt_descendant(adev, &cursor))
continue;
} else if (adev->asic_type < CHIP_VEGA10 &&
(flags & AMDGPU_PTE_VALID)) {
/* No huge page support before GMC v9 */
if (cursor.level != AMDGPU_VM_PTB) {
if (!amdgpu_vm_pt_descendant(adev, &cursor))
return -ENOENT;
continue;
}
} else if (frag < shift) {
/* We can't use this level when the fragment size is
* smaller than the address shift. Go to the next
* child entry and try again.
*/
if (amdgpu_vm_pt_descendant(adev, &cursor))
continue;
} else if (frag >= parent_shift) {
/* If the fragment size is even larger than the parent
* shift we should go up one level and check it again.
*/
if (!amdgpu_vm_pt_ancestor(&cursor))
return -EINVAL;
continue;
}
pt = cursor.entry->bo;
if (!pt) {
/* We need all PDs and PTs for mapping something, */
if (flags & AMDGPU_PTE_VALID)
return -ENOENT;
/* but unmapping something can happen at a higher
* level.
*/
if (!amdgpu_vm_pt_ancestor(&cursor))
return -EINVAL;
pt = cursor.entry->bo;
shift = parent_shift;
frag_end = max(frag_end, ALIGN(frag_start + 1,
1ULL << shift));
}
/* Looks good so far, calculate parameters for the update */
incr = (uint64_t)AMDGPU_GPU_PAGE_SIZE << shift;
mask = amdgpu_vm_pt_entries_mask(adev, cursor.level);
pe_start = ((cursor.pfn >> shift) & mask) * 8;
entry_end = ((uint64_t)mask + 1) << shift;
entry_end += cursor.pfn & ~(entry_end - 1);
entry_end = min(entry_end, end);
do {
struct amdgpu_vm *vm = params->vm;
uint64_t upd_end = min(entry_end, frag_end);
unsigned int nptes = (upd_end - frag_start) >> shift;
uint64_t upd_flags = flags | AMDGPU_PTE_FRAG(frag);
/* This can happen when we set higher level PDs to
* silent to stop fault floods.
*/
nptes = max(nptes, 1u);
trace_amdgpu_vm_update_ptes(params, frag_start, upd_end,
min(nptes, 32u), dst, incr,
upd_flags,
vm->task_info.pid,
vm->immediate.fence_context);
amdgpu_vm_pte_update_flags(params, to_amdgpu_bo_vm(pt),
cursor.level, pe_start, dst,
nptes, incr, upd_flags);
pe_start += nptes * 8;
dst += nptes * incr;
frag_start = upd_end;
if (frag_start >= frag_end) {
/* figure out the next fragment */
amdgpu_vm_pte_fragment(params, frag_start, end,
flags, &frag, &frag_end);
if (frag < shift)
break;
}
} while (frag_start < entry_end);
if (amdgpu_vm_pt_descendant(adev, &cursor)) {
/* Free all child entries.
* Update the tables with the flags and addresses and free up subsequent
* tables in the case of huge pages or freed up areas.
* This is the maximum you can free, because all other page tables are not
* completely covered by the range and so potentially still in use.
*/
while (cursor.pfn < frag_start) {
/* Make sure previous mapping is freed */
if (cursor.entry->bo) {
params->table_freed = true;
amdgpu_vm_pt_free_dfs(adev, params->vm,
&cursor);
}
amdgpu_vm_pt_next(adev, &cursor);
}
} else if (frag >= shift) {
/* or just move on to the next on the same level. */
amdgpu_vm_pt_next(adev, &cursor);
}
}
return 0;
}

2
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
View File

@ -109,7 +109,7 @@ static int amdgpu_vm_sdma_commit(struct amdgpu_vm_update_params *p,
if (p->unlocked) {
struct dma_fence *tmp = dma_fence_get(f);
swap(p->vm->last_unlocked, f);
swap(p->vm->last_unlocked, tmp);
dma_fence_put(tmp);
} else {
amdgpu_bo_fence(p->vm->root.bo, f, true);

2
drivers/gpu/drm/amd/amdgpu/athub_v1_0.c
View File

@ -87,7 +87,7 @@ int athub_v1_0_set_clockgating(struct amdgpu_device *adev,
return 0;
}
void athub_v1_0_get_clockgating(struct amdgpu_device *adev, u32 *flags)
void athub_v1_0_get_clockgating(struct amdgpu_device *adev, u64 *flags)
{
int data;

2
drivers/gpu/drm/amd/amdgpu/athub_v1_0.h
View File

@ -25,6 +25,6 @@
int athub_v1_0_set_clockgating(struct amdgpu_device *adev,
enum amd_clockgating_state state);
void athub_v1_0_get_clockgating(struct amdgpu_device *adev, u32 *flags);
void athub_v1_0_get_clockgating(struct amdgpu_device *adev, u64 *flags);
#endif

2
drivers/gpu/drm/amd/amdgpu/athub_v2_0.c
View File

@ -93,7 +93,7 @@ int athub_v2_0_set_clockgating(struct amdgpu_device *adev,
return 0;
}
void athub_v2_0_get_clockgating(struct amdgpu_device *adev, u32 *flags)
void athub_v2_0_get_clockgating(struct amdgpu_device *adev, u64 *flags)
{
int data;

2
drivers/gpu/drm/amd/amdgpu/athub_v2_0.h
View File

@ -25,6 +25,6 @@
int athub_v2_0_set_clockgating(struct amdgpu_device *adev,
enum amd_clockgating_state state);
void athub_v2_0_get_clockgating(struct amdgpu_device *adev, u32 *flags);
void athub_v2_0_get_clockgating(struct amdgpu_device *adev, u64 *flags);
#endif

2
drivers/gpu/drm/amd/amdgpu/athub_v2_1.c
View File

@ -85,7 +85,7 @@ int athub_v2_1_set_clockgating(struct amdgpu_device *adev,
return 0;
}
void athub_v2_1_get_clockgating(struct amdgpu_device *adev, u32 *flags)
void athub_v2_1_get_clockgating(struct amdgpu_device *adev, u64 *flags)
{
int data;

2
drivers/gpu/drm/amd/amdgpu/athub_v2_1.h
View File

@ -25,6 +25,6 @@
int athub_v2_1_set_clockgating(struct amdgpu_device *adev,
enum amd_clockgating_state state);
void athub_v2_1_get_clockgating(struct amdgpu_device *adev, u32 *flags);
void athub_v2_1_get_clockgating(struct amdgpu_device *adev, u64 *flags);
#endif

21
drivers/gpu/drm/amd/amdgpu/atombios_encoders.c
View File

@ -765,7 +765,6 @@ amdgpu_atombios_encoder_setup_dig_transmitter(struct drm_encoder *encoder, int a
int dp_clock = 0;
int dp_lane_count = 0;
int connector_object_id = 0;
int igp_lane_info = 0;
int dig_encoder = dig->dig_encoder;
int hpd_id = AMDGPU_HPD_NONE;
@ -848,26 +847,6 @@ amdgpu_atombios_encoder_setup_dig_transmitter(struct drm_encoder *encoder, int a
else
args.v1.ucConfig |= ATOM_TRANSMITTER_CONFIG_DIG1_ENCODER;
if ((adev->flags & AMD_IS_APU) &&
(amdgpu_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_UNIPHY)) {
if (is_dp ||
!amdgpu_dig_monitor_is_duallink(encoder, amdgpu_encoder->pixel_clock)) {
if (igp_lane_info & 0x1)
args.v1.ucConfig |= ATOM_TRANSMITTER_CONFIG_LANE_0_3;
else if (igp_lane_info & 0x2)
args.v1.ucConfig |= ATOM_TRANSMITTER_CONFIG_LANE_4_7;
else if (igp_lane_info & 0x4)
args.v1.ucConfig |= ATOM_TRANSMITTER_CONFIG_LANE_8_11;
else if (igp_lane_info & 0x8)
args.v1.ucConfig |= ATOM_TRANSMITTER_CONFIG_LANE_12_15;
} else {
if (igp_lane_info & 0x3)
args.v1.ucConfig |= ATOM_TRANSMITTER_CONFIG_LANE_0_7;
else if (igp_lane_info & 0xc)
args.v1.ucConfig |= ATOM_TRANSMITTER_CONFIG_LANE_8_15;
}
}
if (dig->linkb)
args.v1.ucConfig |= ATOM_TRANSMITTER_CONFIG_LINKB;
else

2
drivers/gpu/drm/amd/amdgpu/df_v1_7.c
View File

@ -99,7 +99,7 @@ static void df_v1_7_update_medium_grain_clock_gating(struct amdgpu_device *adev,
}
static void df_v1_7_get_clockgating_state(struct amdgpu_device *adev,
u32 *flags)
u64 *flags)
{
u32 tmp;

2
drivers/gpu/drm/amd/amdgpu/df_v3_6.c
View File

@ -332,7 +332,7 @@ static void df_v3_6_update_medium_grain_clock_gating(struct amdgpu_device *adev,
}
static void df_v3_6_get_clockgating_state(struct amdgpu_device *adev,
u32 *flags)
u64 *flags)
{
u32 tmp;

4
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
View File

@ -4741,7 +4741,7 @@ static int gfx_v10_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
+ ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
+ ring->pipe;
hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
AMDGPU_RING_PRIO_2 : AMDGPU_RING_PRIO_DEFAULT;
/* type-2 packets are deprecated on MEC, use type-3 instead */
r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
hw_prio, NULL);
@ -8451,7 +8451,7 @@ static int gfx_v10_0_set_clockgating_state(void *handle,
return 0;
}
static void gfx_v10_0_get_clockgating_state(void *handle, u32 *flags)
static void gfx_v10_0_get_clockgating_state(void *handle, u64 *flags)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
int data;

4
drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
View File

@ -1925,7 +1925,7 @@ static int gfx_v8_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
+ ring->pipe;
hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_RING_PRIO_DEFAULT;
AMDGPU_RING_PRIO_2 : AMDGPU_RING_PRIO_DEFAULT;
/* type-2 packets are deprecated on MEC, use type-3 instead */
r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
hw_prio, NULL);
@ -5475,7 +5475,7 @@ static int gfx_v8_0_set_powergating_state(void *handle,
return 0;
}
static void gfx_v8_0_get_clockgating_state(void *handle, u32 *flags)
static void gfx_v8_0_get_clockgating_state(void *handle, u64 *flags)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
int data;

6
drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
View File

@ -1205,6 +1205,8 @@ static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = {
{ 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 },
/* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */
{ 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 },
/* Apple MacBook Pro (15-inch, 2019) Radeon Pro Vega 20 4 GB */
{ 0x1002, 0x69af, 0x106b, 0x019a, 0xc0 },
{ 0, 0, 0, 0, 0 },
};
@ -2274,7 +2276,7 @@ static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
+ ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
+ ring->pipe;
hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
AMDGPU_RING_PRIO_2 : AMDGPU_RING_PRIO_DEFAULT;
/* type-2 packets are deprecated on MEC, use type-3 instead */
return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
hw_prio, NULL);
@ -5231,7 +5233,7 @@ static int gfx_v9_0_set_clockgating_state(void *handle,
return 0;
}
static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags)
static void gfx_v9_0_get_clockgating_state(void *handle, u64 *flags)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
int data;

14
drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c
View File

@ -1930,6 +1930,19 @@ static void gfx_v9_4_2_reset_sq_timeout_status(struct amdgpu_device *adev)
mutex_unlock(&adev->grbm_idx_mutex);
}
static bool gfx_v9_4_2_query_uctl2_poison_status(struct amdgpu_device *adev)
{
u32 status = 0;
struct amdgpu_vmhub *hub;
hub = &adev->vmhub[AMDGPU_GFXHUB_0];
status = RREG32(hub->vm_l2_pro_fault_status);
/* reset page fault status */
WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
return REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, FED);
}
struct amdgpu_ras_block_hw_ops gfx_v9_4_2_ras_ops = {
.ras_error_inject = &gfx_v9_4_2_ras_error_inject,
.query_ras_error_count = &gfx_v9_4_2_query_ras_error_count,
@ -1943,4 +1956,5 @@ struct amdgpu_gfx_ras gfx_v9_4_2_ras = {
.hw_ops = &gfx_v9_4_2_ras_ops,
},
.enable_watchdog_timer = &gfx_v9_4_2_enable_watchdog_timer,
.query_utcl2_poison_status = gfx_v9_4_2_query_uctl2_poison_status,
};

2
drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
View File

@ -1161,7 +1161,7 @@ static int gmc_v10_0_set_clockgating_state(void *handle,
return athub_v2_0_set_clockgating(adev, state);
}
static void gmc_v10_0_get_clockgating_state(void *handle, u32 *flags)
static void gmc_v10_0_get_clockgating_state(void *handle, u64 *flags)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;

2
drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
View File

@ -1690,7 +1690,7 @@ static int gmc_v8_0_set_powergating_state(void *handle,
return 0;
}
static void gmc_v8_0_get_clockgating_state(void *handle, u32 *flags)
static void gmc_v8_0_get_clockgating_state(void *handle, u64 *flags)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
int data;

2
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
View File

@ -1948,7 +1948,7 @@ static int gmc_v9_0_set_clockgating_state(void *handle,
return 0;
}
static void gmc_v9_0_get_clockgating_state(void *handle, u32 *flags)
static void gmc_v9_0_get_clockgating_state(void *handle, u64 *flags)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;

2
drivers/gpu/drm/amd/amdgpu/hdp_v4_0.c
View File

@ -124,7 +124,7 @@ static void hdp_v4_0_update_clock_gating(struct amdgpu_device *adev,
}
static void hdp_v4_0_get_clockgating_state(struct amdgpu_device *adev,
u32 *flags)
u64 *flags)
{
int data;

2
drivers/gpu/drm/amd/amdgpu/hdp_v5_0.c
View File

@ -181,7 +181,7 @@ static void hdp_v5_0_update_clock_gating(struct amdgpu_device *adev,
}
static void hdp_v5_0_get_clockgating_state(struct amdgpu_device *adev,
u32 *flags)
u64 *flags)
{
uint32_t tmp;

74
drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c
View File

@ -26,6 +26,7 @@
#include "soc15.h"
#include "soc15d.h"
#include "jpeg_v2_0.h"
#include "jpeg_v2_5.h"
#include "vcn/vcn_2_5_offset.h"
#include "vcn/vcn_2_5_sh_mask.h"
@ -39,6 +40,7 @@ static void jpeg_v2_5_set_dec_ring_funcs(struct amdgpu_device *adev);
static void jpeg_v2_5_set_irq_funcs(struct amdgpu_device *adev);
static int jpeg_v2_5_set_powergating_state(void *handle,
enum amd_powergating_state state);
static void jpeg_v2_5_set_ras_funcs(struct amdgpu_device *adev);
static int amdgpu_ih_clientid_jpeg[] = {
SOC15_IH_CLIENTID_VCN,
@ -70,6 +72,7 @@ static int jpeg_v2_5_early_init(void *handle)
jpeg_v2_5_set_dec_ring_funcs(adev);
jpeg_v2_5_set_irq_funcs(adev);
jpeg_v2_5_set_ras_funcs(adev);
return 0;
}
@ -730,3 +733,74 @@ const struct amdgpu_ip_block_version jpeg_v2_6_ip_block =
.rev = 0,
.funcs = &jpeg_v2_6_ip_funcs,
};
static uint32_t jpeg_v2_6_query_poison_by_instance(struct amdgpu_device *adev,
uint32_t instance, uint32_t sub_block)
{
uint32_t poison_stat = 0, reg_value = 0;
switch (sub_block) {
case AMDGPU_JPEG_V2_6_JPEG0:
reg_value = RREG32_SOC15(JPEG, instance, mmUVD_RAS_JPEG0_STATUS);
poison_stat = REG_GET_FIELD(reg_value, UVD_RAS_JPEG0_STATUS, POISONED_PF);
break;
case AMDGPU_JPEG_V2_6_JPEG1:
reg_value = RREG32_SOC15(JPEG, instance, mmUVD_RAS_JPEG1_STATUS);
poison_stat = REG_GET_FIELD(reg_value, UVD_RAS_JPEG1_STATUS, POISONED_PF);
break;
default:
break;
}
if (poison_stat)
dev_info(adev->dev, "Poison detected in JPEG%d sub_block%d\n",
instance, sub_block);
return poison_stat;
}
static bool jpeg_v2_6_query_ras_poison_status(struct amdgpu_device *adev)
{
uint32_t inst = 0, sub = 0, poison_stat = 0;
for (inst = 0; inst < adev->jpeg.num_jpeg_inst; inst++)
for (sub = 0; sub < AMDGPU_JPEG_V2_6_MAX_SUB_BLOCK; sub++)
poison_stat +=
jpeg_v2_6_query_poison_by_instance(adev, inst, sub);
return !!poison_stat;
}
const struct amdgpu_ras_block_hw_ops jpeg_v2_6_ras_hw_ops = {
.query_poison_status = jpeg_v2_6_query_ras_poison_status,
};
static struct amdgpu_jpeg_ras jpeg_v2_6_ras = {
.ras_block = {
.hw_ops = &jpeg_v2_6_ras_hw_ops,
},
};
static void jpeg_v2_5_set_ras_funcs(struct amdgpu_device *adev)
{
switch (adev->ip_versions[JPEG_HWIP][0]) {
case IP_VERSION(2, 6, 0):
adev->jpeg.ras = &jpeg_v2_6_ras;
break;
default:
break;
}
if (adev->jpeg.ras) {
amdgpu_ras_register_ras_block(adev, &adev->jpeg.ras->ras_block);
strcpy(adev->jpeg.ras->ras_block.ras_comm.name, "jpeg");
adev->jpeg.ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__JPEG;
adev->jpeg.ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__POISON;
adev->jpeg.ras_if = &adev->jpeg.ras->ras_block.ras_comm;
/* If don't define special ras_late_init function, use default ras_late_init */
if (!adev->jpeg.ras->ras_block.ras_late_init)
adev->jpeg.ras->ras_block.ras_late_init = amdgpu_ras_block_late_init;
}
}

7
drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.h
View File

@ -24,6 +24,13 @@
#ifndef __JPEG_V2_5_H__
#define __JPEG_V2_5_H__
enum amdgpu_jpeg_v2_6_sub_block {
AMDGPU_JPEG_V2_6_JPEG0 = 0,
AMDGPU_JPEG_V2_6_JPEG1,
AMDGPU_JPEG_V2_6_MAX_SUB_BLOCK,
};
extern const struct amdgpu_ip_block_version jpeg_v2_5_ip_block;
extern const struct amdgpu_ip_block_version jpeg_v2_6_ip_block;

2
drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
View File

@ -546,7 +546,7 @@ static int mmhub_v1_0_set_clockgating(struct amdgpu_device *adev,
return 0;
}
static void mmhub_v1_0_get_clockgating(struct amdgpu_device *adev, u32 *flags)
static void mmhub_v1_0_get_clockgating(struct amdgpu_device *adev, u64 *flags)
{
int data, data1;

2
drivers/gpu/drm/amd/amdgpu/mmhub_v1_7.c
View File

@ -542,7 +542,7 @@ static int mmhub_v1_7_set_clockgating(struct amdgpu_device *adev,
return 0;
}
static void mmhub_v1_7_get_clockgating(struct amdgpu_device *adev, u32 *flags)
static void mmhub_v1_7_get_clockgating(struct amdgpu_device *adev, u64 *flags)
{
int data, data1;

2
drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c
View File

@ -682,7 +682,7 @@ static int mmhub_v2_0_set_clockgating(struct amdgpu_device *adev,
return 0;
}
static void mmhub_v2_0_get_clockgating(struct amdgpu_device *adev, u32 *flags)
static void mmhub_v2_0_get_clockgating(struct amdgpu_device *adev, u64 *flags)
{
int data, data1;

2
drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c
View File

@ -577,7 +577,7 @@ static int mmhub_v2_3_set_clockgating(struct amdgpu_device *adev,
return 0;
}
static void mmhub_v2_3_get_clockgating(struct amdgpu_device *adev, u32 *flags)
static void mmhub_v2_3_get_clockgating(struct amdgpu_device *adev, u64 *flags)
{
int data, data1, data2, data3;

2
drivers/gpu/drm/amd/amdgpu/mmhub_v9_4.c
View File

@ -647,7 +647,7 @@ static int mmhub_v9_4_set_clockgating(struct amdgpu_device *adev,
return 0;
}
static void mmhub_v9_4_get_clockgating(struct amdgpu_device *adev, u32 *flags)
static void mmhub_v9_4_get_clockgating(struct amdgpu_device *adev, u64 *flags)
{
int data, data1;

2
drivers/gpu/drm/amd/amdgpu/navi10_ih.c
View File

@ -685,7 +685,7 @@ static int navi10_ih_set_powergating_state(void *handle,
return 0;
}
static void navi10_ih_get_clockgating_state(void *handle, u32 *flags)
static void navi10_ih_get_clockgating_state(void *handle, u64 *flags)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;

2
drivers/gpu/drm/amd/amdgpu/nbio_v2_3.c
View File

@ -278,7 +278,7 @@ static void nbio_v2_3_update_medium_grain_light_sleep(struct amdgpu_device *adev
}
static void nbio_v2_3_get_clockgating_state(struct amdgpu_device *adev,
u32 *flags)
u64 *flags)
{
int data;

2
drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c
View File

@ -210,7 +210,7 @@ static void nbio_v6_1_update_medium_grain_light_sleep(struct amdgpu_device *adev
}
static void nbio_v6_1_get_clockgating_state(struct amdgpu_device *adev,
u32 *flags)
u64 *flags)
{
int data;

2
drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c
View File

@ -205,7 +205,7 @@ static void nbio_v7_0_update_medium_grain_light_sleep(struct amdgpu_device *adev
}
static void nbio_v7_0_get_clockgating_state(struct amdgpu_device *adev,
u32 *flags)
u64 *flags)
{
int data;

2
drivers/gpu/drm/amd/amdgpu/nbio_v7_2.c
View File

@ -306,7 +306,7 @@ static void nbio_v7_2_update_medium_grain_light_sleep(struct amdgpu_device *adev
}
static void nbio_v7_2_get_clockgating_state(struct amdgpu_device *adev,
u32 *flags)
u64 *flags)
{
int data;

2
drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c
View File

@ -273,7 +273,7 @@ static void nbio_v7_4_update_medium_grain_light_sleep(struct amdgpu_device *adev
}
static void nbio_v7_4_get_clockgating_state(struct amdgpu_device *adev,
u32 *flags)
u64 *flags)
{
int data;

2
drivers/gpu/drm/amd/amdgpu/nv.c
View File

@ -1115,7 +1115,7 @@ static int nv_common_set_powergating_state(void *handle,
return 0;
}
static void nv_common_get_clockgating_state(void *handle, u32 *flags)
static void nv_common_get_clockgating_state(void *handle, u64 *flags)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;

2
drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
View File

@ -1535,7 +1535,7 @@ static int sdma_v3_0_set_powergating_state(void *handle,
return 0;
}
static void sdma_v3_0_get_clockgating_state(void *handle, u32 *flags)
static void sdma_v3_0_get_clockgating_state(void *handle, u64 *flags)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
int data;

2
drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
View File

@ -2372,7 +2372,7 @@ static int sdma_v4_0_set_powergating_state(void *handle,
return 0;
}
static void sdma_v4_0_get_clockgating_state(void *handle, u32 *flags)
static void sdma_v4_0_get_clockgating_state(void *handle, u64 *flags)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
int data;

2
drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
View File

@ -1648,7 +1648,7 @@ static int sdma_v5_0_set_powergating_state(void *handle,
return 0;
}
static void sdma_v5_0_get_clockgating_state(void *handle, u32 *flags)
static void sdma_v5_0_get_clockgating_state(void *handle, u64 *flags)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
int data;

2
drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
View File

@ -1645,7 +1645,7 @@ static int sdma_v5_2_set_powergating_state(void *handle,
return 0;
}
static void sdma_v5_2_get_clockgating_state(void *handle, u32 *flags)
static void sdma_v5_2_get_clockgating_state(void *handle, u64 *flags)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
int data;

2
drivers/gpu/drm/amd/amdgpu/smuio_v11_0.c
View File

@ -59,7 +59,7 @@ static void smuio_v11_0_update_rom_clock_gating(struct amdgpu_device *adev, bool
WREG32_SOC15(SMUIO, 0, mmCGTT_ROM_CLK_CTRL0, data);
}
static void smuio_v11_0_get_clock_gating_state(struct amdgpu_device *adev, u32 *flags)
static void smuio_v11_0_get_clock_gating_state(struct amdgpu_device *adev, u64 *flags)
{
u32 data;

2
drivers/gpu/drm/amd/amdgpu/smuio_v11_0_6.c
View File

@ -56,7 +56,7 @@ static void smuio_v11_0_6_update_rom_clock_gating(struct amdgpu_device *adev, bo
WREG32_SOC15(SMUIO, 0, mmCGTT_ROM_CLK_CTRL0, data);
}
static void smuio_v11_0_6_get_clock_gating_state(struct amdgpu_device *adev, u32 *flags)
static void smuio_v11_0_6_get_clock_gating_state(struct amdgpu_device *adev, u64 *flags)
{
u32 data;

2
drivers/gpu/drm/amd/amdgpu/smuio_v13_0.c
View File

@ -58,7 +58,7 @@ static void smuio_v13_0_update_rom_clock_gating(struct amdgpu_device *adev, bool
WREG32_SOC15(SMUIO, 0, regCGTT_ROM_CLK_CTRL0, data);
}
static void smuio_v13_0_get_clock_gating_state(struct amdgpu_device *adev, u32 *flags)
static void smuio_v13_0_get_clock_gating_state(struct amdgpu_device *adev, u64 *flags)
{
u32 data;

2
drivers/gpu/drm/amd/amdgpu/smuio_v9_0.c
View File

@ -56,7 +56,7 @@ static void smuio_v9_0_update_rom_clock_gating(struct amdgpu_device *adev, bool
WREG32_SOC15(SMUIO, 0, mmCGTT_ROM_CLK_CTRL0, data);
}
static void smuio_v9_0_get_clock_gating_state(struct amdgpu_device *adev, u32 *flags)
static void smuio_v9_0_get_clock_gating_state(struct amdgpu_device *adev, u64 *flags)
{
u32 data;

2
drivers/gpu/drm/amd/amdgpu/soc15.c
View File

@ -1419,7 +1419,7 @@ static int soc15_common_set_clockgating_state(void *handle,
return 0;
}
static void soc15_common_get_clockgating_state(void *handle, u32 *flags)
static void soc15_common_get_clockgating_state(void *handle, u64 *flags)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
int data;

106
drivers/gpu/drm/amd/amdgpu/umc_v6_7.c
View File

@ -64,21 +64,62 @@ static inline uint32_t get_umc_v6_7_channel_index(struct amdgpu_device *adev,
return adev->umc.channel_idx_tbl[umc_inst * adev->umc.channel_inst_num + ch_inst];
}
static void umc_v6_7_query_error_status_helper(struct amdgpu_device *adev,
uint64_t mc_umc_status, uint32_t umc_reg_offset)
{
uint32_t mc_umc_addr;
uint64_t reg_value;
if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Deferred) == 1)
dev_info(adev->dev, "Deferred error, no user action is needed.\n");
if (mc_umc_status)
dev_info(adev->dev, "MCA STATUS 0x%llx, umc_reg_offset 0x%x\n", mc_umc_status, umc_reg_offset);
/* print IPID registers value */
mc_umc_addr =
SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_IPIDT0);
reg_value = RREG64_PCIE((mc_umc_addr + umc_reg_offset) * 4);
if (reg_value)
dev_info(adev->dev, "MCA IPID 0x%llx, umc_reg_offset 0x%x\n", reg_value, umc_reg_offset);
/* print SYND registers value */
mc_umc_addr =
SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_SYNDT0);
reg_value = RREG64_PCIE((mc_umc_addr + umc_reg_offset) * 4);
if (reg_value)
dev_info(adev->dev, "MCA SYND 0x%llx, umc_reg_offset 0x%x\n", reg_value, umc_reg_offset);
/* print MISC0 registers value */
mc_umc_addr =
SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_MISC0T0);
reg_value = RREG64_PCIE((mc_umc_addr + umc_reg_offset) * 4);
if (reg_value)
dev_info(adev->dev, "MCA MISC0 0x%llx, umc_reg_offset 0x%x\n", reg_value, umc_reg_offset);
}
static void umc_v6_7_ecc_info_query_correctable_error_count(struct amdgpu_device *adev,
uint32_t umc_inst, uint32_t ch_inst,
unsigned long *error_count)
{
uint64_t mc_umc_status;
uint32_t eccinfo_table_idx;
uint32_t umc_reg_offset;
struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
umc_reg_offset = get_umc_v6_7_reg_offset(adev,
umc_inst, ch_inst);
eccinfo_table_idx = umc_inst * adev->umc.channel_inst_num + ch_inst;
/* check for SRAM correctable error
MCUMC_STATUS is a 64 bit register */
mc_umc_status = ras->umc_ecc.ecc[eccinfo_table_idx].mca_umc_status;
if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 &&
REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1)
REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1) {
*error_count += 1;
umc_v6_7_query_error_status_helper(adev, mc_umc_status, umc_reg_offset);
}
}
static void umc_v6_7_ecc_info_querry_uncorrectable_error_count(struct amdgpu_device *adev,
@ -88,8 +129,6 @@ static void umc_v6_7_ecc_info_querry_uncorrectable_error_count(struct amdgpu_dev
uint64_t mc_umc_status;
uint32_t eccinfo_table_idx;
uint32_t umc_reg_offset;
uint32_t mc_umc_addr;
uint64_t reg_value;
struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
umc_reg_offset = get_umc_v6_7_reg_offset(adev,
@ -106,32 +145,7 @@ static void umc_v6_7_ecc_info_querry_uncorrectable_error_count(struct amdgpu_dev
REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, TCC) == 1)) {
*error_count += 1;
if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Deferred) == 1)
dev_info(adev->dev, "Deferred error, no user action is needed.\n");
if (mc_umc_status)
dev_info(adev->dev, "MCA STATUS 0x%llx, umc_reg_offset 0x%x\n", mc_umc_status, umc_reg_offset);
/* print IPID registers value */
mc_umc_addr =
SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_IPIDT0);
reg_value = RREG64_PCIE((mc_umc_addr + umc_reg_offset) * 4);
if (reg_value)
dev_info(adev->dev, "MCA IPID 0x%llx, umc_reg_offset 0x%x\n", reg_value, umc_reg_offset);
/* print SYND registers value */
mc_umc_addr =
SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_SYNDT0);
reg_value = RREG64_PCIE((mc_umc_addr + umc_reg_offset) * 4);
if (reg_value)
dev_info(adev->dev, "MCA SYND 0x%llx, umc_reg_offset 0x%x\n", reg_value, umc_reg_offset);
/* print MISC0 registers value */
mc_umc_addr =
SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_MISC0T0);
reg_value = RREG64_PCIE((mc_umc_addr + umc_reg_offset) * 4);
if (reg_value)
dev_info(adev->dev, "MCA MISC0 0x%llx, umc_reg_offset 0x%x\n", reg_value, umc_reg_offset);
umc_v6_7_query_error_status_helper(adev, mc_umc_status, umc_reg_offset);
}
}
@ -277,8 +291,11 @@ static void umc_v6_7_query_correctable_error_count(struct amdgpu_device *adev,
MCUMC_STATUS is a 64 bit register */
mc_umc_status = RREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4);
if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 &&
REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1)
REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1) {
*error_count += 1;
umc_v6_7_query_error_status_helper(adev, mc_umc_status, umc_reg_offset);
}
}
static void umc_v6_7_querry_uncorrectable_error_count(struct amdgpu_device *adev,
@ -287,8 +304,6 @@ static void umc_v6_7_querry_uncorrectable_error_count(struct amdgpu_device *adev
{
uint64_t mc_umc_status;
uint32_t mc_umc_status_addr;
uint32_t mc_umc_addr;
uint64_t reg_value;
mc_umc_status_addr =
SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_STATUST0);
@ -303,32 +318,7 @@ static void umc_v6_7_querry_uncorrectable_error_count(struct amdgpu_device *adev
REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, TCC) == 1)) {
*error_count += 1;
if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Deferred) == 1)
dev_info(adev->dev, "Deferred error, no user action is needed.\n");
if (mc_umc_status)
dev_info(adev->dev, "MCA STATUS 0x%llx, umc_reg_offset 0x%x\n", mc_umc_status, umc_reg_offset);
/* print IPID registers value */
mc_umc_addr =
SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_IPIDT0);
reg_value = RREG64_PCIE((mc_umc_addr + umc_reg_offset) * 4);
if (reg_value)
dev_info(adev->dev, "MCA IPID 0x%llx, umc_reg_offset 0x%x\n", reg_value, umc_reg_offset);
/* print SYND registers value */
mc_umc_addr =
SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_SYNDT0);
reg_value = RREG64_PCIE((mc_umc_addr + umc_reg_offset) * 4);
if (reg_value)
dev_info(adev->dev, "MCA SYND 0x%llx, umc_reg_offset 0x%x\n", reg_value, umc_reg_offset);
/* print MISC0 registers value */
mc_umc_addr =
SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_MISC0T0);
reg_value = RREG64_PCIE((mc_umc_addr + umc_reg_offset) * 4);
if (reg_value)
dev_info(adev->dev, "MCA MISC0 0x%llx, umc_reg_offset 0x%x\n", reg_value, umc_reg_offset);
umc_v6_7_query_error_status_helper(adev, mc_umc_status, umc_reg_offset);
}
}

2
drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
View File

@ -833,7 +833,7 @@ static int uvd_v5_0_set_powergating_state(void *handle,
return ret;
}
static void uvd_v5_0_get_clockgating_state(void *handle, u32 *flags)
static void uvd_v5_0_get_clockgating_state(void *handle, u64 *flags)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
int data;

2
drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
View File

@ -1494,7 +1494,7 @@ static int uvd_v6_0_set_powergating_state(void *handle,
return ret;
}
static void uvd_v6_0_get_clockgating_state(void *handle, u32 *flags)
static void uvd_v6_0_get_clockgating_state(void *handle, u64 *flags)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
int data;

2
drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
View File

@ -831,7 +831,7 @@ static int vce_v3_0_set_powergating_state(void *handle,
return ret;
}
static void vce_v3_0_get_clockgating_state(void *handle, u32 *flags)
static void vce_v3_0_get_clockgating_state(void *handle, u64 *flags)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
int data;

71
drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
View File

@ -31,6 +31,7 @@
#include "soc15d.h"
#include "vcn_v2_0.h"
#include "mmsch_v1_0.h"
#include "vcn_v2_5.h"
#include "vcn/vcn_2_5_offset.h"
#include "vcn/vcn_2_5_sh_mask.h"
@ -59,6 +60,7 @@ static int vcn_v2_5_set_powergating_state(void *handle,
static int vcn_v2_5_pause_dpg_mode(struct amdgpu_device *adev,
int inst_idx, struct dpg_pause_state *new_state);
static int vcn_v2_5_sriov_start(struct amdgpu_device *adev);
static void vcn_v2_5_set_ras_funcs(struct amdgpu_device *adev);
static int amdgpu_ih_clientid_vcns[] = {
SOC15_IH_CLIENTID_VCN,
@ -100,6 +102,7 @@ static int vcn_v2_5_early_init(void *handle)
vcn_v2_5_set_dec_ring_funcs(adev);
vcn_v2_5_set_enc_ring_funcs(adev);
vcn_v2_5_set_irq_funcs(adev);
vcn_v2_5_set_ras_funcs(adev);
return 0;
}
@ -1932,3 +1935,71 @@ const struct amdgpu_ip_block_version vcn_v2_6_ip_block =
.rev = 0,
.funcs = &vcn_v2_6_ip_funcs,
};
static uint32_t vcn_v2_6_query_poison_by_instance(struct amdgpu_device *adev,
uint32_t instance, uint32_t sub_block)
{
uint32_t poison_stat = 0, reg_value = 0;
switch (sub_block) {
case AMDGPU_VCN_V2_6_VCPU_VCODEC:
reg_value = RREG32_SOC15(VCN, instance, mmUVD_RAS_VCPU_VCODEC_STATUS);
poison_stat = REG_GET_FIELD(reg_value, UVD_RAS_VCPU_VCODEC_STATUS, POISONED_PF);
break;
default:
break;
}
if (poison_stat)
dev_info(adev->dev, "Poison detected in VCN%d, sub_block%d\n",
instance, sub_block);
return poison_stat;
}
static bool vcn_v2_6_query_poison_status(struct amdgpu_device *adev)
{
uint32_t inst, sub;
uint32_t poison_stat = 0;
for (inst = 0; inst < adev->vcn.num_vcn_inst; inst++)
for (sub = 0; sub < AMDGPU_VCN_V2_6_MAX_SUB_BLOCK; sub++)
poison_stat +=
vcn_v2_6_query_poison_by_instance(adev, inst, sub);
return !!poison_stat;
}
const struct amdgpu_ras_block_hw_ops vcn_v2_6_ras_hw_ops = {
.query_poison_status = vcn_v2_6_query_poison_status,
};
static struct amdgpu_vcn_ras vcn_v2_6_ras = {
.ras_block = {
.hw_ops = &vcn_v2_6_ras_hw_ops,
},
};
static void vcn_v2_5_set_ras_funcs(struct amdgpu_device *adev)
{
switch (adev->ip_versions[VCN_HWIP][0]) {
case IP_VERSION(2, 6, 0):
adev->vcn.ras = &vcn_v2_6_ras;
break;
default:
break;
}
if (adev->vcn.ras) {
amdgpu_ras_register_ras_block(adev, &adev->vcn.ras->ras_block);
strcpy(adev->vcn.ras->ras_block.ras_comm.name, "vcn");
adev->vcn.ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__VCN;
adev->vcn.ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__POISON;
adev->vcn.ras_if = &adev->vcn.ras->ras_block.ras_comm;
/* If don't define special ras_late_init function, use default ras_late_init */
if (!adev->vcn.ras->ras_block.ras_late_init)
adev->vcn.ras->ras_block.ras_late_init = amdgpu_ras_block_late_init;
}
}

6
drivers/gpu/drm/amd/amdgpu/vcn_v2_5.h
View File

@ -24,6 +24,12 @@
#ifndef __VCN_V2_5_H__
#define __VCN_V2_5_H__
enum amdgpu_vcn_v2_6_sub_block {
AMDGPU_VCN_V2_6_VCPU_VCODEC = 0,
AMDGPU_VCN_V2_6_MAX_SUB_BLOCK,
};
extern const struct amdgpu_ip_block_version vcn_v2_5_ip_block;
extern const struct amdgpu_ip_block_version vcn_v2_6_ip_block;

2
drivers/gpu/drm/amd/amdgpu/vi.c
View File

@ -2033,7 +2033,7 @@ static int vi_common_set_powergating_state(void *handle,
return 0;
}
static void vi_common_get_clockgating_state(void *handle, u32 *flags)
static void vi_common_get_clockgating_state(void *handle, u64 *flags)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
int data;

19
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
View File

@ -1146,7 +1146,6 @@ static int kfd_ioctl_map_memory_to_gpu(struct file *filep,
long err = 0;
int i;
uint32_t *devices_arr = NULL;
bool table_freed = false;
if (!args->n_devices) {
pr_debug("Device IDs array empty\n");
@ -1208,7 +1207,7 @@ static int kfd_ioctl_map_memory_to_gpu(struct file *filep,
err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
peer_pdd->dev->adev, (struct kgd_mem *)mem,
peer_pdd->drm_priv, &table_freed);
peer_pdd->drm_priv);
if (err) {
struct pci_dev *pdev = peer_pdd->dev->adev->pdev;
@ -1233,13 +1232,11 @@ static int kfd_ioctl_map_memory_to_gpu(struct file *filep,
}
/* Flush TLBs after waiting for the page table updates to complete */
if (table_freed || !kfd_flush_tlb_after_unmap(dev)) {
for (i = 0; i < args->n_devices; i++) {
peer_pdd = kfd_process_device_data_by_id(p, devices_arr[i]);
if (WARN_ON_ONCE(!peer_pdd))
continue;
kfd_flush_tlb(peer_pdd, TLB_FLUSH_LEGACY);
}
for (i = 0; i < args->n_devices; i++) {
peer_pdd = kfd_process_device_data_by_id(p, devices_arr[i]);
if (WARN_ON_ONCE(!peer_pdd))
continue;
kfd_flush_tlb(peer_pdd, TLB_FLUSH_LEGACY);
}
kfree(devices_arr);
@ -2206,8 +2203,8 @@ static int criu_restore_bo(struct kfd_process *p,
if (IS_ERR(peer_pdd))
return PTR_ERR(peer_pdd);
ret = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(peer->adev, kgd_mem, peer_pdd->drm_priv,
NULL);
ret = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(peer->adev, kgd_mem,
peer_pdd->drm_priv);
if (ret) {
pr_err("Failed to map to gpu %d/%d\n", j, p->n_pdds);
return ret;

4
drivers/gpu/drm/amd/amdkfd/kfd_crat.c
View File

@ -1056,7 +1056,7 @@ static int kfd_parse_subtype_iolink(struct crat_subtype_iolink *iolink,
* table, add corresponded reversed direction link now.
*/
if (props && (iolink->flags & CRAT_IOLINK_FLAGS_BI_DIRECTIONAL)) {
to_dev = kfd_topology_device_by_proximity_domain(id_to);
to_dev = kfd_topology_device_by_proximity_domain_no_lock(id_to);
if (!to_dev)
return -ENODEV;
/* same everything but the other direction */
@ -2225,7 +2225,7 @@ static int kfd_create_vcrat_image_gpu(void *pcrat_image,
*/
if (kdev->hive_id) {
for (nid = 0; nid < proximity_domain; ++nid) {
peer_dev = kfd_topology_device_by_proximity_domain(nid);
peer_dev = kfd_topology_device_by_proximity_domain_no_lock(nid);
if (!peer_dev->gpu)
continue;
if (peer_dev->gpu->hive_id != kdev->hive_id)

152
drivers/gpu/drm/amd/amdkfd/kfd_events.c
View File

@ -128,8 +128,8 @@ static int allocate_event_notification_slot(struct kfd_process *p,
}
/*
* Assumes that p->event_mutex is held and of course that p is not going
* away (current or locked).
* Assumes that p->event_mutex or rcu_readlock is held and of course that p is
* not going away.
*/
static struct kfd_event *lookup_event_by_id(struct kfd_process *p, uint32_t id)
{
@ -251,16 +251,18 @@ static void destroy_event(struct kfd_process *p, struct kfd_event *ev)
struct kfd_event_waiter *waiter;
/* Wake up pending waiters. They will return failure */
spin_lock(&ev->lock);
list_for_each_entry(waiter, &ev->wq.head, wait.entry)
waiter->event = NULL;
WRITE_ONCE(waiter->event, NULL);
wake_up_all(&ev->wq);
spin_unlock(&ev->lock);
if (ev->type == KFD_EVENT_TYPE_SIGNAL ||
ev->type == KFD_EVENT_TYPE_DEBUG)
p->signal_event_count--;
idr_remove(&p->event_idr, ev->event_id);
kfree(ev);
kfree_rcu(ev, rcu);
}
static void destroy_events(struct kfd_process *p)
@ -392,6 +394,7 @@ int kfd_event_create(struct file *devkfd, struct kfd_process *p,
ev->auto_reset = auto_reset;
ev->signaled = false;
spin_lock_init(&ev->lock);
init_waitqueue_head(&ev->wq);
*event_page_offset = 0;
@ -466,6 +469,7 @@ int kfd_criu_restore_event(struct file *devkfd,
ev->auto_reset = ev_priv->auto_reset;
ev->signaled = ev_priv->signaled;
spin_lock_init(&ev->lock);
init_waitqueue_head(&ev->wq);
mutex_lock(&p->event_mutex);
@ -609,13 +613,13 @@ static void set_event(struct kfd_event *ev)
/* Auto reset if the list is non-empty and we're waking
* someone. waitqueue_active is safe here because we're
* protected by the p->event_mutex, which is also held when
* protected by the ev->lock, which is also held when
* updating the wait queues in kfd_wait_on_events.
*/
ev->signaled = !ev->auto_reset || !waitqueue_active(&ev->wq);
list_for_each_entry(waiter, &ev->wq.head, wait.entry)
waiter->activated = true;
WRITE_ONCE(waiter->activated, true);
wake_up_all(&ev->wq);
}
@ -626,16 +630,23 @@ int kfd_set_event(struct kfd_process *p, uint32_t event_id)
int ret = 0;
struct kfd_event *ev;
mutex_lock(&p->event_mutex);
rcu_read_lock();
ev = lookup_event_by_id(p, event_id);
if (!ev) {
ret = -EINVAL;
goto unlock_rcu;
}
spin_lock(&ev->lock);
if (ev && event_can_be_cpu_signaled(ev))
if (event_can_be_cpu_signaled(ev))
set_event(ev);
else
ret = -EINVAL;
mutex_unlock(&p->event_mutex);
spin_unlock(&ev->lock);
unlock_rcu:
rcu_read_unlock();
return ret;
}
@ -650,23 +661,30 @@ int kfd_reset_event(struct kfd_process *p, uint32_t event_id)
int ret = 0;
struct kfd_event *ev;
mutex_lock(&p->event_mutex);
rcu_read_lock();
ev = lookup_event_by_id(p, event_id);
if (!ev) {
ret = -EINVAL;
goto unlock_rcu;
}
spin_lock(&ev->lock);
if (ev && event_can_be_cpu_signaled(ev))
if (event_can_be_cpu_signaled(ev))
reset_event(ev);
else
ret = -EINVAL;
mutex_unlock(&p->event_mutex);
spin_unlock(&ev->lock);
unlock_rcu:
rcu_read_unlock();
return ret;
}
static void acknowledge_signal(struct kfd_process *p, struct kfd_event *ev)
{
page_slots(p->signal_page)[ev->event_id] = UNSIGNALED_EVENT_SLOT;
WRITE_ONCE(page_slots(p->signal_page)[ev->event_id], UNSIGNALED_EVENT_SLOT);
}
static void set_event_from_interrupt(struct kfd_process *p,
@ -674,7 +692,9 @@ static void set_event_from_interrupt(struct kfd_process *p,
{
if (ev && event_can_be_gpu_signaled(ev)) {
acknowledge_signal(p, ev);
spin_lock(&ev->lock);
set_event(ev);
spin_unlock(&ev->lock);
}
}
@ -693,7 +713,7 @@ void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id,
if (!p)
return; /* Presumably process exited. */
mutex_lock(&p->event_mutex);
rcu_read_lock();
if (valid_id_bits)
ev = lookup_signaled_event_by_partial_id(p, partial_id,
@ -721,7 +741,7 @@ void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id,
if (id >= KFD_SIGNAL_EVENT_LIMIT)
break;
if (slots[id] != UNSIGNALED_EVENT_SLOT)
if (READ_ONCE(slots[id]) != UNSIGNALED_EVENT_SLOT)
set_event_from_interrupt(p, ev);
}
} else {
@ -730,14 +750,14 @@ void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id,
* only signaled events from the IDR.
*/
for (id = 0; id < KFD_SIGNAL_EVENT_LIMIT; id++)
if (slots[id] != UNSIGNALED_EVENT_SLOT) {
if (READ_ONCE(slots[id]) != UNSIGNALED_EVENT_SLOT) {
ev = lookup_event_by_id(p, id);
set_event_from_interrupt(p, ev);
}
}
}
mutex_unlock(&p->event_mutex);
rcu_read_unlock();
kfd_unref_process(p);
}
@ -760,7 +780,7 @@ static struct kfd_event_waiter *alloc_event_waiters(uint32_t num_events)
return event_waiters;
}
static int init_event_waiter_get_status(struct kfd_process *p,
static int init_event_waiter(struct kfd_process *p,
struct kfd_event_waiter *waiter,
uint32_t event_id)
{
@ -769,22 +789,15 @@ static int init_event_waiter_get_status(struct kfd_process *p,
if (!ev)
return -EINVAL;
spin_lock(&ev->lock);
waiter->event = ev;
waiter->activated = ev->signaled;
ev->signaled = ev->signaled && !ev->auto_reset;
return 0;
}
static void init_event_waiter_add_to_waitlist(struct kfd_event_waiter *waiter)
{
struct kfd_event *ev = waiter->event;
/* Only add to the wait list if we actually need to
* wait on this event.
*/
if (!waiter->activated)
add_wait_queue(&ev->wq, &waiter->wait);
spin_unlock(&ev->lock);
return 0;
}
/* test_event_condition - Test condition of events being waited for
@ -804,10 +817,10 @@ static uint32_t test_event_condition(bool all, uint32_t num_events,
uint32_t activated_count = 0;
for (i = 0; i < num_events; i++) {
if (!event_waiters[i].event)
if (!READ_ONCE(event_waiters[i].event))
return KFD_IOC_WAIT_RESULT_FAIL;
if (event_waiters[i].activated) {
if (READ_ONCE(event_waiters[i].activated)) {
if (!all)
return KFD_IOC_WAIT_RESULT_COMPLETE;
@ -836,6 +849,8 @@ static int copy_signaled_event_data(uint32_t num_events,
for (i = 0; i < num_events; i++) {
waiter = &event_waiters[i];
event = waiter->event;
if (!event)
return -EINVAL; /* event was destroyed */
if (waiter->activated && event->type == KFD_EVENT_TYPE_MEMORY) {
dst = &data[i].memory_exception_data;
src = &event->memory_exception_data;
@ -846,11 +861,8 @@ static int copy_signaled_event_data(uint32_t num_events,
}
return 0;
}
static long user_timeout_to_jiffies(uint32_t user_timeout_ms)
{
if (user_timeout_ms == KFD_EVENT_TIMEOUT_IMMEDIATE)
@ -874,9 +886,12 @@ static void free_waiters(uint32_t num_events, struct kfd_event_waiter *waiters)
uint32_t i;
for (i = 0; i < num_events; i++)
if (waiters[i].event)
if (waiters[i].event) {
spin_lock(&waiters[i].event->lock);
remove_wait_queue(&waiters[i].event->wq,
&waiters[i].wait);
spin_unlock(&waiters[i].event->lock);
}
kfree(waiters);
}
@ -900,6 +915,9 @@ int kfd_wait_on_events(struct kfd_process *p,
goto out;
}
/* Use p->event_mutex here to protect against concurrent creation and
* destruction of events while we initialize event_waiters.
*/
mutex_lock(&p->event_mutex);
for (i = 0; i < num_events; i++) {
@ -911,8 +929,8 @@ int kfd_wait_on_events(struct kfd_process *p,
goto out_unlock;
}
ret = init_event_waiter_get_status(p, &event_waiters[i],
event_data.event_id);
ret = init_event_waiter(p, &event_waiters[i],
event_data.event_id);
if (ret)
goto out_unlock;
}
@ -930,10 +948,6 @@ int kfd_wait_on_events(struct kfd_process *p,
goto out_unlock;
}
/* Add to wait lists if we need to wait. */
for (i = 0; i < num_events; i++)
init_event_waiter_add_to_waitlist(&event_waiters[i]);
mutex_unlock(&p->event_mutex);
while (true) {
@ -978,14 +992,19 @@ int kfd_wait_on_events(struct kfd_process *p,
}
__set_current_state(TASK_RUNNING);
mutex_lock(&p->event_mutex);
/* copy_signaled_event_data may sleep. So this has to happen
* after the task state is set back to RUNNING.
*
* The event may also have been destroyed after signaling. So
* copy_signaled_event_data also must confirm that the event
* still exists. Therefore this must be under the p->event_mutex
* which is also held when events are destroyed.
*/
if (!ret && *wait_result == KFD_IOC_WAIT_RESULT_COMPLETE)
ret = copy_signaled_event_data(num_events,
event_waiters, events);
mutex_lock(&p->event_mutex);
out_unlock:
free_waiters(num_events, event_waiters);
mutex_unlock(&p->event_mutex);
@ -1044,8 +1063,7 @@ int kfd_event_mmap(struct kfd_process *p, struct vm_area_struct *vma)
}
/*
* Assumes that p->event_mutex is held and of course
* that p is not going away (current or locked).
* Assumes that p is not going away.
*/
static void lookup_events_by_type_and_signal(struct kfd_process *p,
int type, void *event_data)
@ -1057,6 +1075,8 @@ static void lookup_events_by_type_and_signal(struct kfd_process *p,
ev_data = (struct kfd_hsa_memory_exception_data *) event_data;
rcu_read_lock();
id = KFD_FIRST_NONSIGNAL_EVENT_ID;
idr_for_each_entry_continue(&p->event_idr, ev, id)
if (ev->type == type) {
@ -1064,9 +1084,11 @@ static void lookup_events_by_type_and_signal(struct kfd_process *p,
dev_dbg(kfd_device,
"Event found: id %X type %d",
ev->event_id, ev->type);
spin_lock(&ev->lock);
set_event(ev);
if (ev->type == KFD_EVENT_TYPE_MEMORY && ev_data)
ev->memory_exception_data = *ev_data;
spin_unlock(&ev->lock);
}
if (type == KFD_EVENT_TYPE_MEMORY) {
@ -1089,6 +1111,8 @@ static void lookup_events_by_type_and_signal(struct kfd_process *p,
p->lead_thread->pid, p->pasid);
}
}
rcu_read_unlock();
}
#ifdef KFD_SUPPORT_IOMMU_V2
@ -1164,16 +1188,10 @@ void kfd_signal_iommu_event(struct kfd_dev *dev, u32 pasid,
if (KFD_GC_VERSION(dev) != IP_VERSION(9, 1, 0) &&
KFD_GC_VERSION(dev) != IP_VERSION(9, 2, 2) &&
KFD_GC_VERSION(dev) != IP_VERSION(9, 3, 0)) {
mutex_lock(&p->event_mutex);
/* Lookup events by type and signal them */
KFD_GC_VERSION(dev) != IP_VERSION(9, 3, 0))
lookup_events_by_type_and_signal(p, KFD_EVENT_TYPE_MEMORY,
&memory_exception_data);
mutex_unlock(&p->event_mutex);
}
kfd_unref_process(p);
}
#endif /* KFD_SUPPORT_IOMMU_V2 */
@ -1190,12 +1208,7 @@ void kfd_signal_hw_exception_event(u32 pasid)
if (!p)
return; /* Presumably process exited. */
mutex_lock(&p->event_mutex);
/* Lookup events by type and signal them */
lookup_events_by_type_and_signal(p, KFD_EVENT_TYPE_HW_EXCEPTION, NULL);
mutex_unlock(&p->event_mutex);
kfd_unref_process(p);
}
@ -1231,16 +1244,19 @@ void kfd_signal_vm_fault_event(struct kfd_dev *dev, u32 pasid,
info->prot_write ? 1 : 0;
memory_exception_data.failure.imprecise = 0;
}
mutex_lock(&p->event_mutex);
rcu_read_lock();
id = KFD_FIRST_NONSIGNAL_EVENT_ID;
idr_for_each_entry_continue(&p->event_idr, ev, id)
if (ev->type == KFD_EVENT_TYPE_MEMORY) {
spin_lock(&ev->lock);
ev->memory_exception_data = memory_exception_data;
set_event(ev);
spin_unlock(&ev->lock);
}
mutex_unlock(&p->event_mutex);
rcu_read_unlock();
kfd_unref_process(p);
}
@ -1274,22 +1290,28 @@ void kfd_signal_reset_event(struct kfd_dev *dev)
continue;
}
mutex_lock(&p->event_mutex);
rcu_read_lock();
id = KFD_FIRST_NONSIGNAL_EVENT_ID;
idr_for_each_entry_continue(&p->event_idr, ev, id) {
if (ev->type == KFD_EVENT_TYPE_HW_EXCEPTION) {
spin_lock(&ev->lock);
ev->hw_exception_data = hw_exception_data;
ev->hw_exception_data.gpu_id = user_gpu_id;
set_event(ev);
spin_unlock(&ev->lock);
}
if (ev->type == KFD_EVENT_TYPE_MEMORY &&
reset_cause == KFD_HW_EXCEPTION_ECC) {
spin_lock(&ev->lock);
ev->memory_exception_data = memory_exception_data;
ev->memory_exception_data.gpu_id = user_gpu_id;
set_event(ev);
spin_unlock(&ev->lock);
}
}
mutex_unlock(&p->event_mutex);
rcu_read_unlock();
}
srcu_read_unlock(&kfd_processes_srcu, idx);
}
@ -1322,19 +1344,25 @@ void kfd_signal_poison_consumed_event(struct kfd_dev *dev, u32 pasid)
memory_exception_data.gpu_id = user_gpu_id;
memory_exception_data.failure.imprecise = true;
mutex_lock(&p->event_mutex);
rcu_read_lock();
idr_for_each_entry_continue(&p->event_idr, ev, id) {
if (ev->type == KFD_EVENT_TYPE_HW_EXCEPTION) {
spin_lock(&ev->lock);
ev->hw_exception_data = hw_exception_data;
set_event(ev);
spin_unlock(&ev->lock);
}
if (ev->type == KFD_EVENT_TYPE_MEMORY) {
spin_lock(&ev->lock);
ev->memory_exception_data = memory_exception_data;
set_event(ev);
spin_unlock(&ev->lock);
}
}
mutex_unlock(&p->event_mutex);
rcu_read_unlock();
/* user application will handle SIGBUS signal */
send_sig(SIGBUS, p->lead_thread, 0);

3
drivers/gpu/drm/amd/amdkfd/kfd_events.h
View File

@ -59,6 +59,7 @@ struct kfd_event {
int type;
spinlock_t lock;
wait_queue_head_t wq; /* List of event waiters. */
/* Only for signal events. */
@ -69,6 +70,8 @@ struct kfd_event {
struct kfd_hsa_memory_exception_data memory_exception_data;
struct kfd_hsa_hw_exception_data hw_exception_data;
};
struct rcu_head rcu; /* for asynchronous kfree_rcu */
};
#define KFD_EVENT_TIMEOUT_IMMEDIATE 0

49
drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
View File

@ -91,28 +91,34 @@ enum SQ_INTERRUPT_ERROR_TYPE {
#define KFD_SQ_INT_DATA__ERR_TYPE__SHIFT 20
static void event_interrupt_poison_consumption(struct kfd_dev *dev,
uint16_t pasid, uint16_t source_id)
uint16_t pasid, uint16_t client_id)
{
int ret = -EINVAL;
int old_poison, ret = -EINVAL;
struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
if (!p)
return;
/* all queues of a process will be unmapped in one time */
if (atomic_read(&p->poison)) {
kfd_unref_process(p);
return;
}
atomic_set(&p->poison, 1);
old_poison = atomic_cmpxchg(&p->poison, 0, 1);
kfd_unref_process(p);
if (old_poison)
return;
switch (source_id) {
case SOC15_INTSRC_SQ_INTERRUPT_MSG:
switch (client_id) {
case SOC15_IH_CLIENTID_SE0SH:
case SOC15_IH_CLIENTID_SE1SH:
case SOC15_IH_CLIENTID_SE2SH:
case SOC15_IH_CLIENTID_SE3SH:
case SOC15_IH_CLIENTID_UTCL2:
ret = kfd_dqm_evict_pasid(dev->dqm, pasid);
break;
case SOC15_INTSRC_SDMA_ECC:
case SOC15_IH_CLIENTID_SDMA0:
case SOC15_IH_CLIENTID_SDMA1:
case SOC15_IH_CLIENTID_SDMA2:
case SOC15_IH_CLIENTID_SDMA3:
case SOC15_IH_CLIENTID_SDMA4:
break;
default:
break;
}
@ -122,10 +128,17 @@ static void event_interrupt_poison_consumption(struct kfd_dev *dev,
/* resetting queue passes, do page retirement without gpu reset
* resetting queue fails, fallback to gpu reset solution
*/
if (!ret)
if (!ret) {
dev_warn(dev->adev->dev,
"RAS poison consumption, unmap queue flow succeeded: client id %d\n",
client_id);
amdgpu_amdkfd_ras_poison_consumption_handler(dev->adev, false);
else
} else {
dev_warn(dev->adev->dev,
"RAS poison consumption, fall back to gpu reset flow: client id %d\n",
client_id);
amdgpu_amdkfd_ras_poison_consumption_handler(dev->adev, true);
}
}
static bool event_interrupt_isr_v9(struct kfd_dev *dev,
@ -270,7 +283,7 @@ static void event_interrupt_wq_v9(struct kfd_dev *dev,
sq_intr_err);
if (sq_intr_err != SQ_INTERRUPT_ERROR_TYPE_ILLEGAL_INST &&
sq_intr_err != SQ_INTERRUPT_ERROR_TYPE_MEMVIOL) {
event_interrupt_poison_consumption(dev, pasid, source_id);
event_interrupt_poison_consumption(dev, pasid, client_id);
return;
}
break;
@ -291,7 +304,7 @@ static void event_interrupt_wq_v9(struct kfd_dev *dev,
if (source_id == SOC15_INTSRC_SDMA_TRAP) {
kfd_signal_event_interrupt(pasid, context_id0 & 0xfffffff, 28);
} else if (source_id == SOC15_INTSRC_SDMA_ECC) {
event_interrupt_poison_consumption(dev, pasid, source_id);
event_interrupt_poison_consumption(dev, pasid, client_id);
return;
}
} else if (client_id == SOC15_IH_CLIENTID_VMC ||
@ -300,6 +313,12 @@ static void event_interrupt_wq_v9(struct kfd_dev *dev,
struct kfd_vm_fault_info info = {0};
uint16_t ring_id = SOC15_RING_ID_FROM_IH_ENTRY(ih_ring_entry);
if (client_id == SOC15_IH_CLIENTID_UTCL2 &&
amdgpu_amdkfd_ras_query_utcl2_poison_status(dev->adev)) {
event_interrupt_poison_consumption(dev, pasid, client_id);
return;
}
info.vmid = vmid;
info.mc_id = client_id;
info.page_addr = ih_ring_entry[4] |

11
drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c
View File

@ -146,15 +146,24 @@ static void interrupt_wq(struct work_struct *work)
struct kfd_dev *dev = container_of(work, struct kfd_dev,
interrupt_work);
uint32_t ih_ring_entry[KFD_MAX_RING_ENTRY_SIZE];
long start_jiffies = jiffies;
if (dev->device_info.ih_ring_entry_size > sizeof(ih_ring_entry)) {
dev_err_once(dev->adev->dev, "Ring entry too small\n");
return;
}
while (dequeue_ih_ring_entry(dev, ih_ring_entry))
while (dequeue_ih_ring_entry(dev, ih_ring_entry)) {
dev->device_info.event_interrupt_class->interrupt_wq(dev,
ih_ring_entry);
if (jiffies - start_jiffies > HZ) {
/* If we spent more than a second processing signals,
* reschedule the worker to avoid soft-lockup warnings
*/
queue_work(dev->ih_wq, &dev->interrupt_work);
break;
}
}
}
bool interrupt_is_wanted(struct kfd_dev *dev,

3
drivers/gpu/drm/amd/amdkfd/kfd_priv.h
View File

@ -706,6 +706,7 @@ struct kfd_process_device {
/* VM context for GPUVM allocations */
struct file *drm_file;
void *drm_priv;
atomic64_t tlb_seq;
/* GPUVM allocations storage */
struct idr alloc_idr;
@ -1016,6 +1017,8 @@ int kfd_topology_add_device(struct kfd_dev *gpu);
int kfd_topology_remove_device(struct kfd_dev *gpu);
struct kfd_topology_device *kfd_topology_device_by_proximity_domain(
uint32_t proximity_domain);
struct kfd_topology_device *kfd_topology_device_by_proximity_domain_no_lock(
uint32_t proximity_domain);
struct kfd_topology_device *kfd_topology_device_by_id(uint32_t gpu_id);
struct kfd_dev *kfd_device_by_id(uint32_t gpu_id);
struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev);

13
drivers/gpu/drm/amd/amdkfd/kfd_process.c
View File

@ -722,7 +722,7 @@ static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,
goto err_alloc_mem;
err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->adev, *mem,
pdd->drm_priv, NULL);
pdd->drm_priv);
if (err)
goto err_map_mem;
@ -1560,6 +1560,7 @@ int kfd_process_device_init_vm(struct kfd_process_device *pdd,
return ret;
}
pdd->drm_priv = drm_file->private_data;
atomic64_set(&pdd->tlb_seq, 0);
ret = kfd_process_device_reserve_ib_mem(pdd);
if (ret)
@ -1949,8 +1950,18 @@ int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process,
void kfd_flush_tlb(struct kfd_process_device *pdd, enum TLB_FLUSH_TYPE type)
{
struct amdgpu_vm *vm = drm_priv_to_vm(pdd->drm_priv);
uint64_t tlb_seq = amdgpu_vm_tlb_seq(vm);
struct kfd_dev *dev = pdd->dev;
/*
* It can be that we race and lose here, but that is extremely unlikely
* and the worst thing which could happen is that we flush the changes
* into the TLB once more which is harmless.
*/
if (atomic64_xchg(&pdd->tlb_seq, tlb_seq) == tlb_seq)
return;
if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
/* Nothing to flush until a VMID is assigned, which
* only happens when the first queue is created.

37
drivers/gpu/drm/amd/amdkfd/kfd_svm.c
View File

@ -1188,9 +1188,9 @@ svm_range_unmap_from_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
pr_debug("[0x%llx 0x%llx]\n", start, last);
return amdgpu_vm_bo_update_mapping(adev, adev, vm, false, true, NULL,
start, last, init_pte_value, 0,
NULL, NULL, fence, NULL);
return amdgpu_vm_update_range(adev, vm, false, true, true, NULL, start,
last, init_pte_value, 0, 0, NULL, NULL,
fence);
}
static int
@ -1243,7 +1243,6 @@ svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
{
struct amdgpu_device *adev = pdd->dev->adev;
struct amdgpu_vm *vm = drm_priv_to_vm(pdd->drm_priv);
bool table_freed = false;
uint64_t pte_flags;
unsigned long last_start;
int last_domain;
@ -1278,13 +1277,12 @@ svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
(last_domain == SVM_RANGE_VRAM_DOMAIN) ? 1 : 0,
pte_flags);
r = amdgpu_vm_bo_update_mapping(adev, bo_adev, vm, false, false,
NULL, last_start,
prange->start + i, pte_flags,
last_start - prange->start,
NULL, dma_addr,
&vm->last_update,
&table_freed);
r = amdgpu_vm_update_range(adev, vm, false, false, false, NULL,
last_start, prange->start + i,
pte_flags,
last_start - prange->start,
bo_adev ? bo_adev->vm_manager.vram_base_offset : 0,
NULL, dma_addr, &vm->last_update);
for (j = last_start - prange->start; j <= i; j++)
dma_addr[j] |= last_domain;
@ -1306,8 +1304,6 @@ svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
if (fence)
*fence = dma_fence_get(vm->last_update);
if (table_freed)
kfd_flush_tlb(pdd, TLB_FLUSH_LEGACY);
out:
return r;
}
@ -1363,6 +1359,8 @@ svm_range_map_to_gpus(struct svm_range *prange, unsigned long offset,
break;
}
}
kfd_flush_tlb(pdd, TLB_FLUSH_LEGACY);
}
return r;
@ -1372,7 +1370,7 @@ struct svm_validate_context {
struct kfd_process *process;
struct svm_range *prange;
bool intr;
unsigned long bitmap[MAX_GPU_INSTANCE];
DECLARE_BITMAP(bitmap, MAX_GPU_INSTANCE);
struct ttm_validate_buffer tv[MAX_GPU_INSTANCE];
struct list_head validate_list;
struct ww_acquire_ctx ticket;
@ -2687,11 +2685,6 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
pr_debug("kfd process not founded pasid 0x%x\n", pasid);
return 0;
}
if (!p->xnack_enabled) {
pr_debug("XNACK not enabled for pasid 0x%x\n", pasid);
r = -EFAULT;
goto out;
}
svms = &p->svms;
pr_debug("restoring svms 0x%p fault address 0x%llx\n", svms, addr);
@ -2702,6 +2695,12 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
goto out;
}
if (!p->xnack_enabled) {
pr_debug("XNACK not enabled for pasid 0x%x\n", pasid);
r = -EFAULT;
goto out;
}
/* p->lead_thread is available as kfd_process_wq_release flush the work
* before releasing task ref.
*/

83
drivers/gpu/drm/amd/amdkfd/kfd_topology.c
View File

@ -46,22 +46,32 @@ static struct list_head topology_device_list;
static struct kfd_system_properties sys_props;
static DECLARE_RWSEM(topology_lock);
static atomic_t topology_crat_proximity_domain;
static uint32_t topology_crat_proximity_domain;
struct kfd_topology_device *kfd_topology_device_by_proximity_domain(
struct kfd_topology_device *kfd_topology_device_by_proximity_domain_no_lock(
uint32_t proximity_domain)
{
struct kfd_topology_device *top_dev;
struct kfd_topology_device *device = NULL;
down_read(&topology_lock);
list_for_each_entry(top_dev, &topology_device_list, list)
if (top_dev->proximity_domain == proximity_domain) {
device = top_dev;
break;
}
return device;
}
struct kfd_topology_device *kfd_topology_device_by_proximity_domain(
uint32_t proximity_domain)
{
struct kfd_topology_device *device = NULL;
down_read(&topology_lock);
device = kfd_topology_device_by_proximity_domain_no_lock(
proximity_domain);
up_read(&topology_lock);
return device;
@ -1060,7 +1070,7 @@ int kfd_topology_init(void)
down_write(&topology_lock);
kfd_topology_update_device_list(&temp_topology_device_list,
&topology_device_list);
atomic_set(&topology_crat_proximity_domain, sys_props.num_devices-1);
topology_crat_proximity_domain = sys_props.num_devices-1;
ret = kfd_topology_update_sysfs();
up_write(&topology_lock);
@ -1295,8 +1305,6 @@ int kfd_topology_add_device(struct kfd_dev *gpu)
pr_debug("Adding new GPU (ID: 0x%x) to topology\n", gpu_id);
proximity_domain = atomic_inc_return(&topology_crat_proximity_domain);
/* Include the CPU in xGMI hive if xGMI connected by assigning it the hive ID. */
if (gpu->hive_id && gpu->adev->gmc.xgmi.connected_to_cpu) {
struct kfd_topology_device *top_dev;
@ -1321,12 +1329,16 @@ int kfd_topology_add_device(struct kfd_dev *gpu)
*/
dev = kfd_assign_gpu(gpu);
if (!dev) {
down_write(&topology_lock);
proximity_domain = ++topology_crat_proximity_domain;
res = kfd_create_crat_image_virtual(&crat_image, &image_size,
COMPUTE_UNIT_GPU, gpu,
proximity_domain);
if (res) {
pr_err("Error creating VCRAT for GPU (ID: 0x%x)\n",
gpu_id);
topology_crat_proximity_domain--;
return res;
}
res = kfd_parse_crat_table(crat_image,
@ -1335,10 +1347,10 @@ int kfd_topology_add_device(struct kfd_dev *gpu)
if (res) {
pr_err("Error parsing VCRAT for GPU (ID: 0x%x)\n",
gpu_id);
topology_crat_proximity_domain--;
goto err;
}
down_write(&topology_lock);
kfd_topology_update_device_list(&temp_topology_device_list,
&topology_device_list);
@ -1485,25 +1497,78 @@ int kfd_topology_add_device(struct kfd_dev *gpu)
return res;
}
/**
* kfd_topology_update_io_links() - Update IO links after device removal.
* @proximity_domain: Proximity domain value of the dev being removed.
*
* The topology list currently is arranged in increasing order of
* proximity domain.
*
* Two things need to be done when a device is removed:
* 1. All the IO links to this device need to be removed.
* 2. All nodes after the current device node need to move
* up once this device node is removed from the topology
* list. As a result, the proximity domain values for
* all nodes after the node being deleted reduce by 1.
* This would also cause the proximity domain values for
* io links to be updated based on new proximity domain
* values.
*
* Context: The caller must hold write topology_lock.
*/
static void kfd_topology_update_io_links(int proximity_domain)
{
struct kfd_topology_device *dev;
struct kfd_iolink_properties *iolink, *tmp;
list_for_each_entry(dev, &topology_device_list, list) {
if (dev->proximity_domain > proximity_domain)
dev->proximity_domain--;
list_for_each_entry_safe(iolink, tmp, &dev->io_link_props, list) {
/*
* If there is an io link to the dev being deleted
* then remove that IO link also.
*/
if (iolink->node_to == proximity_domain) {
list_del(&iolink->list);
dev->io_link_count--;
dev->node_props.io_links_count--;
} else if (iolink->node_from > proximity_domain) {
iolink->node_from--;
} else if (iolink->node_to > proximity_domain) {
iolink->node_to--;
}
}
}
}
int kfd_topology_remove_device(struct kfd_dev *gpu)
{
struct kfd_topology_device *dev, *tmp;
uint32_t gpu_id;
int res = -ENODEV;
int i = 0;
down_write(&topology_lock);
list_for_each_entry_safe(dev, tmp, &topology_device_list, list)
list_for_each_entry_safe(dev, tmp, &topology_device_list, list) {
if (dev->gpu == gpu) {
gpu_id = dev->gpu_id;
kfd_remove_sysfs_node_entry(dev);
kfd_release_topology_device(dev);
sys_props.num_devices--;
kfd_topology_update_io_links(i);
topology_crat_proximity_domain = sys_props.num_devices-1;
sys_props.generation_count++;
res = 0;
if (kfd_topology_update_sysfs() < 0)
kfd_topology_release_sysfs();
break;
}
i++;
}
up_write(&topology_lock);

8
drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
View File

@ -9251,7 +9251,7 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state,
&bundle->flip_addrs[planes_count].address,
afb->tmz_surface, false);
DRM_DEBUG_ATOMIC("plane: id=%d dcc_en=%d\n",
drm_dbg_state(state->dev, "plane: id=%d dcc_en=%d\n",
new_plane_state->plane->index,
bundle->plane_infos[planes_count].dcc.enable);
@ -9285,7 +9285,7 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state,
dc_plane,
bundle->flip_addrs[planes_count].flip_timestamp_in_us);
DRM_DEBUG_ATOMIC("%s Flipping to hi: 0x%x, low: 0x%x\n",
drm_dbg_state(state->dev, "%s Flipping to hi: 0x%x, low: 0x%x\n",
__func__,
bundle->flip_addrs[planes_count].address.grph.addr.high_part,
bundle->flip_addrs[planes_count].address.grph.addr.low_part);
@ -9627,7 +9627,7 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state)
dm_new_crtc_state = to_dm_crtc_state(new_crtc_state);
dm_old_crtc_state = to_dm_crtc_state(old_crtc_state);
DRM_DEBUG_ATOMIC(
drm_dbg_state(state->dev,
"amdgpu_crtc id:%d crtc_state_flags: enable:%d, active:%d, "
"planes_changed:%d, mode_changed:%d,active_changed:%d,"
"connectors_changed:%d\n",
@ -10331,7 +10331,7 @@ static int dm_update_crtc_state(struct amdgpu_display_manager *dm,
if (!drm_atomic_crtc_needs_modeset(new_crtc_state))
goto skip_modeset;
DRM_DEBUG_ATOMIC(
drm_dbg_state(state->dev,
"amdgpu_crtc id:%d crtc_state_flags: enable:%d, active:%d, "
"planes_changed:%d, mode_changed:%d,active_changed:%d,"
"connectors_changed:%d\n",

37
drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
View File

@ -3491,6 +3491,40 @@ DEFINE_SHOW_ATTRIBUTE(mst_topo);
DEFINE_DEBUGFS_ATTRIBUTE(visual_confirm_fops, visual_confirm_get,
visual_confirm_set, "%llu\n");
/*
* Sets the DC skip_detection_link_training debug option from the given string.
* Example usage: echo 1 > /sys/kernel/debug/dri/0/amdgpu_skip_detection_link_training
*/
static int skip_detection_link_training_set(void *data, u64 val)
{
struct amdgpu_device *adev = data;
if (val == 0)
adev->dm.dc->debug.skip_detection_link_training = false;
else
adev->dm.dc->debug.skip_detection_link_training = true;
return 0;
}
/*
* Reads the DC skip_detection_link_training debug option value into the given buffer.
* Example usage: cat /sys/kernel/debug/dri/0/amdgpu_dm_skip_detection_link_training
*/
static int skip_detection_link_training_get(void *data, u64 *val)
{
struct amdgpu_device *adev = data;
*val = adev->dm.dc->debug.skip_detection_link_training;
return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(skip_detection_link_training_fops,
skip_detection_link_training_get,
skip_detection_link_training_set, "%llu\n");
/*
* Dumps the DCC_EN bit for each pipe.
* Example usage: cat /sys/kernel/debug/dri/0/amdgpu_dm_dcc_en
@ -3584,6 +3618,9 @@ void dtn_debugfs_init(struct amdgpu_device *adev)
debugfs_create_file_unsafe("amdgpu_dm_visual_confirm", 0644, root, adev,
&visual_confirm_fops);
debugfs_create_file_unsafe("amdgpu_dm_skip_detection_link_training", 0644, root, adev,
&skip_detection_link_training_fops);
debugfs_create_file_unsafe("amdgpu_dm_dmub_tracebuffer", 0644, root,
adev, &dmub_tracebuffer_fops);

2
drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
View File

@ -90,7 +90,7 @@ enum dc_edid_status dm_helpers_parse_edid_caps(
{
struct amdgpu_dm_connector *aconnector = link->priv;
struct drm_connector *connector = &aconnector->base;
struct edid *edid_buf = (struct edid *) edid->raw_edid;
struct edid *edid_buf = edid ? (struct edid *) edid->raw_edid : NULL;
struct cea_sad *sads;
int sad_count = -1;
int sadb_count = -1;

9
drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c
View File

@ -27,6 +27,7 @@
#include "dc.h"
#include "dm_helpers.h"
#include "amdgpu_dm.h"
#include "modules/power/power_helpers.h"
#ifdef CONFIG_DRM_AMD_DC_DCN
static bool link_supports_psrsu(struct dc_link *link)
@ -39,6 +40,9 @@ static bool link_supports_psrsu(struct dc_link *link)
if (dc->ctx->dce_version < DCN_VERSION_3_1)
return false;
if (!is_psr_su_specific_panel(link))
return false;
if (!link->dpcd_caps.alpm_caps.bits.AUX_WAKE_ALPM_CAP ||
!link->dpcd_caps.psr_info.psr_dpcd_caps.bits.Y_COORDINATE_REQUIRED)
return false;
@ -79,7 +83,10 @@ void amdgpu_dm_set_psr_caps(struct dc_link *link)
link->psr_settings.psr_feature_enabled = true;
}
DRM_INFO("PSR support:%d\n", link->psr_settings.psr_feature_enabled);
DRM_INFO("PSR support %d, DC PSR ver %d, sink PSR ver %d\n",
link->psr_settings.psr_feature_enabled,
link->psr_settings.psr_version,
link->dpcd_caps.psr_info.psr_version);
}

3
drivers/gpu/drm/amd/display/dc/bios/bios_parser_common.c
View File

@ -213,6 +213,9 @@ static enum connector_id connector_id_from_bios_object_id(
case CONNECTOR_OBJECT_ID_MXM:
id = CONNECTOR_ID_MXM;
break;
case CONNECTOR_OBJECT_ID_USBC:
id = CONNECTOR_ID_USBC;
break;
default:
id = CONNECTOR_ID_UNKNOWN;
break;

3
drivers/gpu/drm/amd/display/dc/bios/command_table.c
View File

@ -522,7 +522,8 @@ static enum bp_result transmitter_control_v2(
*/
params.acConfig.ucEncoderSel = 1;
if (CONNECTOR_ID_DISPLAY_PORT == connector_id)
if (CONNECTOR_ID_DISPLAY_PORT == connector_id
|| CONNECTOR_ID_USBC == connector_id)
/* Bit4: DP connector flag
* =0 connector is none-DP connector
* =1 connector is DP connector

2
drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.c
View File

@ -374,7 +374,7 @@ void dce_clock_read_ss_info(struct clk_mgr_internal *clk_mgr_dce)
clk_mgr_dce->dprefclk_ss_percentage =
info.spread_spectrum_percentage;
}
if (clk_mgr_dce->base.ctx->dc->debug.ignore_dpref_ss)
if (clk_mgr_dce->base.ctx->dc->config.ignore_dpref_ss)
clk_mgr_dce->dprefclk_ss_percentage = 0;
}
}

Some files were not shown because too many files have changed in this diff Show More