drm/amdgpu: add df perfmon regs and funcs for xgmi

v6: Squash in warning fix (Colin Ian King)
v5: Fix warnings (Alex)
v4: fixed mixed delaration and code warnings and minor errors
v3: exposing df funcs in amdgpu_df_funcs in amdgpu.h
v2: moving permonctl/perfmonctr from default to offset

- adding df perfmonctl and perfmonctr registers for df counters
- adding df funcs to set perfmonctl and get perfmonctr for
df and xgmi counters
- exposing df funcs in amdgpu_df_funcs

Signed-off-by: Jonathan Kim <jonathan.kim@amd.com>
Reviewed-by: Evan Quan <evan.quan@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

drivers/gpu/drm/amd/amdgpu/amdgpu.h

@@ -210,6 +210,7 @@ struct amdgpu_irq_src;
 struct amdgpu_fpriv;
 struct amdgpu_bo_va_mapping;
 struct amdgpu_atif;
+struct kfd_vm_fault_info;
 
 enum amdgpu_cp_irq {
 	AMDGPU_CP_IRQ_GFX_EOP = 0,
@@ -688,6 +689,12 @@ struct amdgpu_df_funcs {
 				      u32 *flags);
 	void (*enable_ecc_force_par_wr_rmw)(struct amdgpu_device *adev,
 					    bool enable);
+	int (*pmc_start)(struct amdgpu_device *adev, uint64_t config,
+					 int is_enable);
+	int (*pmc_stop)(struct amdgpu_device *adev, uint64_t config,
+					 int is_disable);
+	void (*pmc_get_count)(struct amdgpu_device *adev, uint64_t config,
+					 uint64_t *count);
 };
 /* Define the HW IP blocks will be used in driver , add more if necessary */
 enum amd_hw_ip_block_type {
@@ -1096,6 +1103,9 @@ void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
 					     const u32 array_size);
 
 bool amdgpu_device_is_px(struct drm_device *dev);
+bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
+				      struct amdgpu_device *peer_adev);
+
 /* atpx handler */
 #if defined(CONFIG_VGA_SWITCHEROO)
 void amdgpu_register_atpx_handler(void);

drivers/gpu/drm/amd/amdgpu/df_v3_6.c

@@ -105,6 +105,431 @@ static void df_v3_6_get_clockgating_state(struct amdgpu_device *adev,
 		*flags |= AMD_CG_SUPPORT_DF_MGCG;
 }
 
+/* hold counter assignment per gpu struct */
+struct df_v3_6_event_mask {
+		struct amdgpu_device gpu;
+		uint64_t config_assign_mask[AMDGPU_DF_MAX_COUNTERS];
+};
+
+/* get assigned df perfmon ctr as int */
+static void df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
+				      uint64_t config,
+				      int *counter)
+{
+	struct df_v3_6_event_mask *mask;
+	int i;
+
+	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
+
+	for (i = 0; i < AMDGPU_DF_MAX_COUNTERS; i++) {
+		if ((config & 0x0FFFFFFUL) == mask->config_assign_mask[i]) {
+			*counter = i;
+			return;
+		}
+	}
+}
+
+/* get address based on counter assignment */
+static void df_v3_6_pmc_get_addr(struct amdgpu_device *adev,
+				 uint64_t config,
+				 int is_ctrl,
+				 uint32_t *lo_base_addr,
+				 uint32_t *hi_base_addr)
+{
+
+	int target_cntr = -1;
+
+	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
+
+	if (target_cntr < 0)
+		return;
+
+	switch (target_cntr) {
+
+	case 0:
+		*lo_base_addr = is_ctrl ? smnPerfMonCtlLo0 : smnPerfMonCtrLo0;
+		*hi_base_addr = is_ctrl ? smnPerfMonCtlHi0 : smnPerfMonCtrHi0;
+		break;
+	case 1:
+		*lo_base_addr = is_ctrl ? smnPerfMonCtlLo1 : smnPerfMonCtrLo1;
+		*hi_base_addr = is_ctrl ? smnPerfMonCtlHi1 : smnPerfMonCtrHi1;
+		break;
+	case 2:
+		*lo_base_addr = is_ctrl ? smnPerfMonCtlLo2 : smnPerfMonCtrLo2;
+		*hi_base_addr = is_ctrl ? smnPerfMonCtlHi2 : smnPerfMonCtrHi2;
+		break;
+	case 3:
+		*lo_base_addr = is_ctrl ? smnPerfMonCtlLo3 : smnPerfMonCtrLo3;
+		*hi_base_addr = is_ctrl ? smnPerfMonCtlHi3 : smnPerfMonCtrHi3;
+		break;
+
+	}
+
+}
+
+/* get read counter address */
+static void df_v3_6_pmc_get_read_settings(struct amdgpu_device *adev,
+					  uint64_t config,
+					  uint32_t *lo_base_addr,
+					  uint32_t *hi_base_addr)
+{
+	df_v3_6_pmc_get_addr(adev, config, 0, lo_base_addr, hi_base_addr);
+}
+
+/* get control counter settings i.e. address and values to set */
+static void df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev,
+					  uint64_t config,
+					  uint32_t *lo_base_addr,
+					  uint32_t *hi_base_addr,
+					  uint32_t *lo_val,
+					  uint32_t *hi_val)
+{
+
+	uint32_t eventsel, instance, unitmask;
+	uint32_t es_5_0, es_13_0, es_13_6, es_13_12, es_11_8, es_7_0;
+
+	df_v3_6_pmc_get_addr(adev, config, 1, lo_base_addr, hi_base_addr);
+
+	if (lo_val == NULL || hi_val == NULL)
+		return;
+
+	if ((*lo_base_addr == 0) || (*hi_base_addr == 0)) {
+		DRM_ERROR("DF PMC addressing not retrived! Lo: %x, Hi: %x",
+				*lo_base_addr, *hi_base_addr);
+		return;
+	}
+
+	eventsel = GET_EVENT(config);
+	instance = GET_INSTANCE(config);
+	unitmask = GET_UNITMASK(config);
+
+	es_5_0 = eventsel & 0x3FUL;
+	es_13_6 = instance;
+	es_13_0 = (es_13_6 << 6) + es_5_0;
+	es_13_12 = (es_13_0 & 0x03000UL) >> 12;
+	es_11_8 = (es_13_0 & 0x0F00UL) >> 8;
+	es_7_0 = es_13_0 & 0x0FFUL;
+	*lo_val = (es_7_0 & 0xFFUL) | ((unitmask & 0x0FUL) << 8);
+	*hi_val = (es_11_8 | ((es_13_12)<<(29)));
+}
+
+/* assign df performance counters for read */
+static int df_v3_6_pmc_assign_cntr(struct amdgpu_device *adev,
+				   uint64_t config,
+				   int *is_assigned)
+{
+
+	struct df_v3_6_event_mask *mask;
+	int i, target_cntr;
+
+	target_cntr = -1;
+
+	*is_assigned = 0;
+
+	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
+
+	if (target_cntr >= 0) {
+		*is_assigned = 1;
+		return 0;
+	}
+
+	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
+
+	for (i = 0; i < AMDGPU_DF_MAX_COUNTERS; i++) {
+		if (mask->config_assign_mask[i] == 0ULL) {
+			mask->config_assign_mask[i] = config & 0x0FFFFFFUL;
+			return 0;
+		}
+	}
+
+	return -ENOSPC;
+}
+
+/* release performance counter */
+static void df_v3_6_pmc_release_cntr(struct amdgpu_device *adev,
+				     uint64_t config)
+{
+
+	struct df_v3_6_event_mask *mask;
+	int target_cntr;
+
+	target_cntr = -1;
+
+	df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
+
+	mask = container_of(adev, struct df_v3_6_event_mask, gpu);
+
+	if (target_cntr >= 0)
+		mask->config_assign_mask[target_cntr] = 0ULL;
+
+}
+
+/*
+ * get xgmi link counters via programmable data fabric (df) counters (max 4)
+ * using cake tx event.
+ *
+ * @adev -> amdgpu device
+ * @instance-> currently cake has 2 links to poll on vega20
+ * @count -> counters to pass
+ *
+ */
+
+static void df_v3_6_get_xgmi_link_cntr(struct amdgpu_device *adev,
+				       int instance,
+				       uint64_t *count)
+{
+	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
+	uint64_t config;
+
+	config = GET_INSTANCE_CONFIG(instance);
+
+	df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
+				      &hi_base_addr);
+
+	if ((lo_base_addr == 0) || (hi_base_addr == 0))
+		return;
+
+	lo_val = RREG32_PCIE(lo_base_addr);
+	hi_val = RREG32_PCIE(hi_base_addr);
+
+	*count  = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
+}
+
+/*
+ * reset xgmi link counters
+ *
+ * @adev -> amdgpu device
+ * @instance-> currently cake has 2 links to poll on vega20
+ *
+ */
+static void df_v3_6_reset_xgmi_link_cntr(struct amdgpu_device *adev,
+					 int instance)
+{
+	uint32_t lo_base_addr, hi_base_addr;
+	uint64_t config;
+
+	config = 0ULL | (0x7ULL) | ((0x46ULL + instance) << 8) | (0x2 << 16);
+
+	df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
+				      &hi_base_addr);
+
+	if ((lo_base_addr == 0) || (hi_base_addr == 0))
+		return;
+
+	WREG32_PCIE(lo_base_addr, 0UL);
+	WREG32_PCIE(hi_base_addr, 0UL);
+}
+
+/*
+ * add xgmi link counters
+ *
+ * @adev -> amdgpu device
+ * @instance-> currently cake has 2 links to poll on vega20
+ *
+ */
+
+static int df_v3_6_add_xgmi_link_cntr(struct amdgpu_device *adev,
+				      int instance)
+{
+	uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
+	uint64_t config;
+	int ret, is_assigned;
+
+	if (instance < 0 || instance > 1)
+		return -EINVAL;
+
+	config = GET_INSTANCE_CONFIG(instance);
+
+	ret = df_v3_6_pmc_assign_cntr(adev, config, &is_assigned);
+
+	if (ret || is_assigned)
+		return ret;
+
+	df_v3_6_pmc_get_ctrl_settings(adev,
+			config,
+			&lo_base_addr,
+			&hi_base_addr,
+			&lo_val,
+			&hi_val);
+
+	WREG32_PCIE(lo_base_addr, lo_val);
+	WREG32_PCIE(hi_base_addr, hi_val);
+
+	return ret;
+}
+
+
+/*
+ * start xgmi link counters
+ *
+ * @adev -> amdgpu device
+ * @instance-> currently cake has 2 links to poll on vega20
+ * @is_enable -> either resume or assign event via df perfmon
+ *
+ */
+
+static int df_v3_6_start_xgmi_link_cntr(struct amdgpu_device *adev,
+					int instance,
+					int is_enable)
+{
+	uint32_t lo_base_addr, hi_base_addr, lo_val;
+	uint64_t config;
+	int ret;
+
+	if (instance < 0 || instance > 1)
+		return -EINVAL;
+
+	if (is_enable) {
+
+		ret = df_v3_6_add_xgmi_link_cntr(adev, instance);
+
+		if (ret)
+			return ret;
+
+	} else {
+
+		config = GET_INSTANCE_CONFIG(instance);
+
+		df_v3_6_pmc_get_ctrl_settings(adev,
+				config,
+				&lo_base_addr,
+				&hi_base_addr,
+				NULL,
+				NULL);
+
+		if (lo_base_addr == 0)
+			return -EINVAL;
+
+		lo_val = RREG32_PCIE(lo_base_addr);
+
+		WREG32_PCIE(lo_base_addr, lo_val | (1ULL << 22));
+
+		ret = 0;
+	}
+
+	return ret;
+
+}
+
+/*
+ * start xgmi link counters
+ *
+ * @adev -> amdgpu device
+ * @instance-> currently cake has 2 links to poll on vega20
+ * @is_enable -> either pause or unassign event via df perfmon
+ *
+ */
+
+static int df_v3_6_stop_xgmi_link_cntr(struct amdgpu_device *adev,
+				       int instance,
+				       int is_disable)
+{
+
+	uint32_t lo_base_addr, hi_base_addr, lo_val;
+	uint64_t config;
+
+	config = GET_INSTANCE_CONFIG(instance);
+
+	if (is_disable) {
+		df_v3_6_reset_xgmi_link_cntr(adev, instance);
+		df_v3_6_pmc_release_cntr(adev, config);
+	} else {
+
+		df_v3_6_pmc_get_ctrl_settings(adev,
+				config,
+				&lo_base_addr,
+				&hi_base_addr,
+				NULL,
+				NULL);
+
+		if ((lo_base_addr == 0) || (hi_base_addr == 0))
+			return -EINVAL;
+
+		lo_val = RREG32_PCIE(lo_base_addr);
+
+		WREG32_PCIE(lo_base_addr, lo_val & ~(1ULL << 22));
+	}
+
+	return 0;
+}
+
+static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
+			     int is_enable)
+{
+	int xgmi_tx_link, ret = 0;
+
+	switch (adev->asic_type) {
+	case CHIP_VEGA20:
+		xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
+					: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
+
+		if (xgmi_tx_link >= 0)
+			ret = df_v3_6_start_xgmi_link_cntr(adev, xgmi_tx_link,
+						      is_enable);
+
+		if (ret)
+			return ret;
+
+		ret = 0;
+		break;
+	default:
+		break;
+	}
+
+	return ret;
+}
+
+static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
+			    int is_disable)
+{
+	int xgmi_tx_link, ret = 0;
+
+	switch (adev->asic_type) {
+	case CHIP_VEGA20:
+			xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
+				: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
+
+			if (xgmi_tx_link >= 0) {
+				ret = df_v3_6_stop_xgmi_link_cntr(adev,
+								  xgmi_tx_link,
+								  is_disable);
+				if (ret)
+					return ret;
+			}
+
+			ret = 0;
+			break;
+	default:
+		break;
+	}
+
+	return ret;
+}
+
+static void df_v3_6_pmc_get_count(struct amdgpu_device *adev,
+				  uint64_t config,
+				  uint64_t *count)
+{
+
+	int xgmi_tx_link;
+
+	switch (adev->asic_type) {
+	case CHIP_VEGA20:
+		xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
+					: (IS_DF_XGMI_1_TX(config) ? 1 : -1);
+
+		if (xgmi_tx_link >= 0) {
+			df_v3_6_reset_xgmi_link_cntr(adev, xgmi_tx_link);
+			df_v3_6_get_xgmi_link_cntr(adev, xgmi_tx_link, count);
+		}
+
+		break;
+	default:
+		break;
+	}
+
+}
+
 const struct amdgpu_df_funcs df_v3_6_funcs = {
 	.init = df_v3_6_init,
 	.enable_broadcast_mode = df_v3_6_enable_broadcast_mode,
@@ -113,4 +538,7 @@ const struct amdgpu_df_funcs df_v3_6_funcs = {
 	.update_medium_grain_clock_gating =
 			df_v3_6_update_medium_grain_clock_gating,
 	.get_clockgating_state = df_v3_6_get_clockgating_state,
+	.pmc_start = df_v3_6_pmc_start,
+	.pmc_stop = df_v3_6_pmc_stop,
+	.pmc_get_count = df_v3_6_pmc_get_count
 };

drivers/gpu/drm/amd/amdgpu/df_v3_6.h

@@ -35,6 +35,23 @@ enum DF_V3_6_MGCG {
 	DF_V3_6_MGCG_ENABLE_63_CYCLE_DELAY = 15
 };
 
+/* Defined in global_features.h as FTI_PERFMON_VISIBLE */
+#define AMDGPU_DF_MAX_COUNTERS		4
+
+/* get flags from df perfmon config */
+#define GET_EVENT(x)			(x & 0xFFUL)
+#define GET_INSTANCE(x)			((x >> 8) & 0xFFUL)
+#define GET_UNITMASK(x)			((x >> 16) & 0xFFUL)
+#define GET_INSTANCE_CONFIG(x)		(0ULL | (0x07ULL) \
+					| ((0x046ULL + x) << 8) \
+					| (0x02 << 16))
+
+/* df event conf macros */
+#define IS_DF_XGMI_0_TX(x) (GET_EVENT(x) == 0x7 \
+		&& GET_INSTANCE(x) == 0x46 && GET_UNITMASK(x) == 0x2)
+#define IS_DF_XGMI_1_TX(x) (GET_EVENT(x) == 0x7 \
+		&& GET_INSTANCE(x) == 0x47 && GET_UNITMASK(x) == 0x2)
+
 extern const struct amdgpu_df_funcs df_v3_6_funcs;
 
 #endif

drivers/gpu/drm/amd/include/asic_reg/df/df_3_6_offset.h

@@ -30,4 +30,22 @@
 #define mmDF_CS_UMC_AON0_DramBaseAddress0								0x0044
 #define mmDF_CS_UMC_AON0_DramBaseAddress0_BASE_IDX							0
 
+#define smnPerfMonCtlLo0					0x01d440UL
+#define smnPerfMonCtlHi0					0x01d444UL
+#define smnPerfMonCtlLo1					0x01d450UL
+#define smnPerfMonCtlHi1					0x01d454UL
+#define smnPerfMonCtlLo2					0x01d460UL
+#define smnPerfMonCtlHi2					0x01d464UL
+#define smnPerfMonCtlLo3					0x01d470UL
+#define smnPerfMonCtlHi3					0x01d474UL
+
+#define smnPerfMonCtrLo0					0x01d448UL
+#define smnPerfMonCtrHi0					0x01d44cUL
+#define smnPerfMonCtrLo1					0x01d458UL
+#define smnPerfMonCtrHi1					0x01d45cUL
+#define smnPerfMonCtrLo2					0x01d468UL
+#define smnPerfMonCtrHi2					0x01d46cUL
+#define smnPerfMonCtrLo3					0x01d478UL
+#define smnPerfMonCtrHi3					0x01d47cUL
+
 #endif