From dc4e96ce2dceb649224ee84f83592aac8c54c9b7 Mon Sep 17 00:00:00 2001 From: Steve Wise Date: Sat, 28 Aug 2010 13:35:05 +0000 Subject: [PATCH 1/5] RDMA/cxgb3: Don't exceed the max HW CQ depth The max depth supported by T3 is 64K entries. This fixes a bug introduced in commit 9918b28d ("RDMA/cxgb3: Increase the max CQ depth") that causes stalls and possibly crashes in large MPI clusters. Signed-off-by: Steve Wise Cc: Signed-off-by: Roland Dreier --- drivers/infiniband/hw/cxgb3/cxio_hal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/cxgb3/cxio_hal.h b/drivers/infiniband/hw/cxgb3/cxio_hal.h index 8f0caf7d4482..78fbe9ffe7f0 100644 --- a/drivers/infiniband/hw/cxgb3/cxio_hal.h +++ b/drivers/infiniband/hw/cxgb3/cxio_hal.h @@ -53,7 +53,7 @@ #define T3_MAX_PBL_SIZE 256 #define T3_MAX_RQ_SIZE 1024 #define T3_MAX_QP_DEPTH (T3_MAX_RQ_SIZE-1) -#define T3_MAX_CQ_DEPTH 262144 +#define T3_MAX_CQ_DEPTH 65536 #define T3_MAX_NUM_STAG (1<<15) #define T3_MAX_MR_SIZE 0x100000000ULL #define T3_PAGESIZE_MASK 0xffff000 /* 4KB-128MB */ From 70c9db0fdfd703781c3b8c2caf9287806f642e02 Mon Sep 17 00:00:00 2001 From: Chien Tung Date: Tue, 7 Sep 2010 16:31:20 +0000 Subject: [PATCH 2/5] RDMA/nes: Write correct register write to set TX pause param Setting TX pause param writes to the wrong register location causing the adapter to hang. Correct the define used to write the reigster. Addresses: https://bugs.openfabrics.org/show_bug.cgi?id=2116 Reported-by: Shiri Franchi Signed-off-by: Chien Tung Signed-off-by: Roland Dreier --- drivers/infiniband/hw/nes/nes_nic.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/hw/nes/nes_nic.c b/drivers/infiniband/hw/nes/nes_nic.c index 6dfdd49cdbcf..10560c796fd6 100644 --- a/drivers/infiniband/hw/nes/nes_nic.c +++ b/drivers/infiniband/hw/nes/nes_nic.c @@ -1446,14 +1446,14 @@ static int nes_netdev_set_pauseparam(struct net_device *netdev, NES_IDX_MAC_TX_CONFIG + (nesdev->mac_index*0x200)); u32temp |= NES_IDX_MAC_TX_CONFIG_ENABLE_PAUSE; nes_write_indexed(nesdev, - NES_IDX_MAC_TX_CONFIG_ENABLE_PAUSE + (nesdev->mac_index*0x200), u32temp); + NES_IDX_MAC_TX_CONFIG + (nesdev->mac_index*0x200), u32temp); nesdev->disable_tx_flow_control = 0; } else if ((et_pauseparam->tx_pause == 0) && (nesdev->disable_tx_flow_control == 0)) { u32temp = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG + (nesdev->mac_index*0x200)); u32temp &= ~NES_IDX_MAC_TX_CONFIG_ENABLE_PAUSE; nes_write_indexed(nesdev, - NES_IDX_MAC_TX_CONFIG_ENABLE_PAUSE + (nesdev->mac_index*0x200), u32temp); + NES_IDX_MAC_TX_CONFIG + (nesdev->mac_index*0x200), u32temp); nesdev->disable_tx_flow_control = 1; } if ((et_pauseparam->rx_pause == 1) && (nesdev->disable_rx_flow_control == 1)) { From dae58728dc64e9ad71c40ac90b463bff6ecce271 Mon Sep 17 00:00:00 2001 From: Faisal Latif Date: Sat, 14 Aug 2010 21:04:56 +0000 Subject: [PATCH 3/5] RDMA/nes: Fix double CLOSE event indication crash During a stress testing in a large cluster, multiple close event are detected and BUG() is hit in the iWARP core. The cause is that the active node gave up while waiting for an MPA response from the peer and tried to close the connection by sending RST. The passive node driver receives the RST but is waiting for MPA response from the user. When the MPA accept is received, the driver offloads the connection and sends a CLOSE event. The driver gets an AE indicating RESET received and also sends a CLOSE event, hitting a BUG(). Fix this by correcting RESET handling and sending CLOSE events. Signed-off-by: Faisal Latif Signed-off-by: Roland Dreier --- drivers/infiniband/hw/nes/nes_cm.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/drivers/infiniband/hw/nes/nes_cm.c b/drivers/infiniband/hw/nes/nes_cm.c index 443cea55daac..61e0efd4ccfb 100644 --- a/drivers/infiniband/hw/nes/nes_cm.c +++ b/drivers/infiniband/hw/nes/nes_cm.c @@ -502,7 +502,9 @@ int schedule_nes_timer(struct nes_cm_node *cm_node, struct sk_buff *skb, static void nes_retrans_expired(struct nes_cm_node *cm_node) { struct iw_cm_id *cm_id = cm_node->cm_id; - switch (cm_node->state) { + enum nes_cm_node_state state = cm_node->state; + cm_node->state = NES_CM_STATE_CLOSED; + switch (state) { case NES_CM_STATE_SYN_RCVD: case NES_CM_STATE_CLOSING: rem_ref_cm_node(cm_node->cm_core, cm_node); @@ -511,7 +513,6 @@ static void nes_retrans_expired(struct nes_cm_node *cm_node) case NES_CM_STATE_FIN_WAIT1: if (cm_node->cm_id) cm_id->rem_ref(cm_id); - cm_node->state = NES_CM_STATE_CLOSED; send_reset(cm_node, NULL); break; default: @@ -1439,9 +1440,6 @@ static void handle_rst_pkt(struct nes_cm_node *cm_node, struct sk_buff *skb, break; case NES_CM_STATE_MPAREQ_RCVD: passive_state = atomic_add_return(1, &cm_node->passive_state); - if (passive_state == NES_SEND_RESET_EVENT) - create_event(cm_node, NES_CM_EVENT_RESET); - cm_node->state = NES_CM_STATE_CLOSED; dev_kfree_skb_any(skb); break; case NES_CM_STATE_ESTABLISHED: @@ -1456,6 +1454,7 @@ static void handle_rst_pkt(struct nes_cm_node *cm_node, struct sk_buff *skb, case NES_CM_STATE_CLOSED: drop_packet(skb); break; + case NES_CM_STATE_FIN_WAIT2: case NES_CM_STATE_FIN_WAIT1: case NES_CM_STATE_LAST_ACK: cm_node->cm_id->rem_ref(cm_node->cm_id); @@ -2777,6 +2776,12 @@ int nes_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) return -EINVAL; } + passive_state = atomic_add_return(1, &cm_node->passive_state); + if (passive_state == NES_SEND_RESET_EVENT) { + rem_ref_cm_node(cm_node->cm_core, cm_node); + return -ECONNRESET; + } + /* associate the node with the QP */ nesqp->cm_node = (void *)cm_node; cm_node->nesqp = nesqp; @@ -2979,9 +2984,6 @@ int nes_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) printk(KERN_ERR "%s[%u] OFA CM event_handler returned, " "ret=%d\n", __func__, __LINE__, ret); - passive_state = atomic_add_return(1, &cm_node->passive_state); - if (passive_state == NES_SEND_RESET_EVENT) - create_event(cm_node, NES_CM_EVENT_RESET); return 0; } From 67d70721151726286763209ecadc3fce86abfdce Mon Sep 17 00:00:00 2001 From: Faisal Latif Date: Sat, 14 Aug 2010 21:05:04 +0000 Subject: [PATCH 4/5] RDMA/nes: Change state to closing after FIN When the driver receives an AE for FIN received, it closes the connection without changing the state of the connection in the hardware to closing. By changing the state to closing, hardware will do a normal close sequence. Signed-off-by: Faisal Latif Signed-off-by: Roland Dreier --- drivers/infiniband/hw/nes/nes_hw.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/nes/nes_hw.c b/drivers/infiniband/hw/nes/nes_hw.c index f8233c851c69..ba93a8be6bf6 100644 --- a/drivers/infiniband/hw/nes/nes_hw.c +++ b/drivers/infiniband/hw/nes/nes_hw.c @@ -3468,6 +3468,18 @@ static void nes_process_iwarp_aeqe(struct nes_device *nesdev, return; /* Ignore it, wait for close complete */ if (atomic_inc_return(&nesqp->close_timer_started) == 1) { + if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) && + (nesqp->ibqp_state == IB_QPS_RTS)) { + spin_lock_irqsave(&nesqp->lock, flags); + nesqp->hw_iwarp_state = iwarp_state; + nesqp->hw_tcp_state = tcp_state; + nesqp->last_aeq = async_event_id; + next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING; + nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING; + spin_unlock_irqrestore(&nesqp->lock, flags); + nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0); + nes_cm_disconn(nesqp); + } nesqp->cm_id->add_ref(nesqp->cm_id); schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp, NES_TIMER_TYPE_CLOSE, 1, 0); @@ -3477,7 +3489,6 @@ static void nes_process_iwarp_aeqe(struct nes_device *nesdev, nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), async_event_id, nesqp->last_aeq, tcp_state); } - break; case NES_AEQE_AEID_LLP_CLOSE_COMPLETE: if (nesqp->term_flags) { From 29da03b9d1c6f24548d98cebda1e15a25d80ee1b Mon Sep 17 00:00:00 2001 From: Faisal Latif Date: Wed, 1 Sep 2010 15:43:11 +0000 Subject: [PATCH 5/5] RDMA/nes: Fix hang with modified FIN handling on A0 cards Changing state to CLOSING when FIN is received causes A0 cards to hang. Fix this by checking for A0 cards in FIN handling. Signed-off-by: Faisal Latif Signed-off-by: Roland Dreier --- drivers/infiniband/hw/nes/nes_hw.c | 3 ++- drivers/infiniband/hw/nes/nes_hw.h | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/nes/nes_hw.c b/drivers/infiniband/hw/nes/nes_hw.c index ba93a8be6bf6..1980a461c499 100644 --- a/drivers/infiniband/hw/nes/nes_hw.c +++ b/drivers/infiniband/hw/nes/nes_hw.c @@ -3469,7 +3469,8 @@ static void nes_process_iwarp_aeqe(struct nes_device *nesdev, if (atomic_inc_return(&nesqp->close_timer_started) == 1) { if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) && - (nesqp->ibqp_state == IB_QPS_RTS)) { + (nesqp->ibqp_state == IB_QPS_RTS) && + ((nesadapter->eeprom_version >> 16) != NES_A0)) { spin_lock_irqsave(&nesqp->lock, flags); nesqp->hw_iwarp_state = iwarp_state; nesqp->hw_tcp_state = tcp_state; diff --git a/drivers/infiniband/hw/nes/nes_hw.h b/drivers/infiniband/hw/nes/nes_hw.h index aa9183db32b1..1204c3432b63 100644 --- a/drivers/infiniband/hw/nes/nes_hw.h +++ b/drivers/infiniband/hw/nes/nes_hw.h @@ -45,6 +45,7 @@ #define NES_PHY_TYPE_KR 9 #define NES_MULTICAST_PF_MAX 8 +#define NES_A0 3 enum pci_regs { NES_INT_STAT = 0x0000,