2017-03-15 14:40:00 +08:00
|
|
|
/* Broadcom FlexRM Mailbox Driver
|
|
|
|
*
|
|
|
|
* Copyright (C) 2017 Broadcom
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
|
|
* published by the Free Software Foundation.
|
|
|
|
*
|
|
|
|
* Each Broadcom FlexSparx4 offload engine is implemented as an
|
|
|
|
* extension to Broadcom FlexRM ring manager. The FlexRM ring
|
|
|
|
* manager provides a set of rings which can be used to submit
|
|
|
|
* work to a FlexSparx4 offload engine.
|
|
|
|
*
|
|
|
|
* This driver creates a mailbox controller using a set of FlexRM
|
|
|
|
* rings where each mailbox channel represents a separate FlexRM ring.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <asm/barrier.h>
|
|
|
|
#include <asm/byteorder.h>
|
2017-08-01 18:35:51 +08:00
|
|
|
#include <linux/atomic.h>
|
2017-08-01 18:35:53 +08:00
|
|
|
#include <linux/bitmap.h>
|
2017-08-01 18:35:51 +08:00
|
|
|
#include <linux/debugfs.h>
|
2017-03-15 14:40:00 +08:00
|
|
|
#include <linux/delay.h>
|
|
|
|
#include <linux/device.h>
|
|
|
|
#include <linux/dma-mapping.h>
|
|
|
|
#include <linux/dmapool.h>
|
|
|
|
#include <linux/err.h>
|
|
|
|
#include <linux/interrupt.h>
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/mailbox_controller.h>
|
|
|
|
#include <linux/mailbox_client.h>
|
|
|
|
#include <linux/mailbox/brcm-message.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/msi.h>
|
|
|
|
#include <linux/of_address.h>
|
|
|
|
#include <linux/of_irq.h>
|
|
|
|
#include <linux/platform_device.h>
|
|
|
|
#include <linux/spinlock.h>
|
|
|
|
|
|
|
|
/* ====== FlexRM register defines ===== */
|
|
|
|
|
|
|
|
/* FlexRM configuration */
|
|
|
|
#define RING_REGS_SIZE 0x10000
|
|
|
|
#define RING_DESC_SIZE 8
|
|
|
|
#define RING_DESC_INDEX(offset) \
|
|
|
|
((offset) / RING_DESC_SIZE)
|
|
|
|
#define RING_DESC_OFFSET(index) \
|
|
|
|
((index) * RING_DESC_SIZE)
|
|
|
|
#define RING_MAX_REQ_COUNT 1024
|
|
|
|
#define RING_BD_ALIGN_ORDER 12
|
|
|
|
#define RING_BD_ALIGN_CHECK(addr) \
|
|
|
|
(!((addr) & ((0x1 << RING_BD_ALIGN_ORDER) - 1)))
|
|
|
|
#define RING_BD_TOGGLE_INVALID(offset) \
|
|
|
|
(((offset) >> RING_BD_ALIGN_ORDER) & 0x1)
|
|
|
|
#define RING_BD_TOGGLE_VALID(offset) \
|
|
|
|
(!RING_BD_TOGGLE_INVALID(offset))
|
|
|
|
#define RING_BD_DESC_PER_REQ 32
|
|
|
|
#define RING_BD_DESC_COUNT \
|
|
|
|
(RING_MAX_REQ_COUNT * RING_BD_DESC_PER_REQ)
|
|
|
|
#define RING_BD_SIZE \
|
|
|
|
(RING_BD_DESC_COUNT * RING_DESC_SIZE)
|
|
|
|
#define RING_CMPL_ALIGN_ORDER 13
|
|
|
|
#define RING_CMPL_DESC_COUNT RING_MAX_REQ_COUNT
|
|
|
|
#define RING_CMPL_SIZE \
|
|
|
|
(RING_CMPL_DESC_COUNT * RING_DESC_SIZE)
|
|
|
|
#define RING_VER_MAGIC 0x76303031
|
|
|
|
|
|
|
|
/* Per-Ring register offsets */
|
|
|
|
#define RING_VER 0x000
|
|
|
|
#define RING_BD_START_ADDR 0x004
|
|
|
|
#define RING_BD_READ_PTR 0x008
|
|
|
|
#define RING_BD_WRITE_PTR 0x00c
|
|
|
|
#define RING_BD_READ_PTR_DDR_LS 0x010
|
|
|
|
#define RING_BD_READ_PTR_DDR_MS 0x014
|
|
|
|
#define RING_CMPL_START_ADDR 0x018
|
|
|
|
#define RING_CMPL_WRITE_PTR 0x01c
|
|
|
|
#define RING_NUM_REQ_RECV_LS 0x020
|
|
|
|
#define RING_NUM_REQ_RECV_MS 0x024
|
|
|
|
#define RING_NUM_REQ_TRANS_LS 0x028
|
|
|
|
#define RING_NUM_REQ_TRANS_MS 0x02c
|
|
|
|
#define RING_NUM_REQ_OUTSTAND 0x030
|
|
|
|
#define RING_CONTROL 0x034
|
|
|
|
#define RING_FLUSH_DONE 0x038
|
|
|
|
#define RING_MSI_ADDR_LS 0x03c
|
|
|
|
#define RING_MSI_ADDR_MS 0x040
|
|
|
|
#define RING_MSI_CONTROL 0x048
|
|
|
|
#define RING_BD_READ_PTR_DDR_CONTROL 0x04c
|
|
|
|
#define RING_MSI_DATA_VALUE 0x064
|
|
|
|
|
|
|
|
/* Register RING_BD_START_ADDR fields */
|
|
|
|
#define BD_LAST_UPDATE_HW_SHIFT 28
|
|
|
|
#define BD_LAST_UPDATE_HW_MASK 0x1
|
|
|
|
#define BD_START_ADDR_VALUE(pa) \
|
|
|
|
((u32)((((dma_addr_t)(pa)) >> RING_BD_ALIGN_ORDER) & 0x0fffffff))
|
|
|
|
#define BD_START_ADDR_DECODE(val) \
|
|
|
|
((dma_addr_t)((val) & 0x0fffffff) << RING_BD_ALIGN_ORDER)
|
|
|
|
|
|
|
|
/* Register RING_CMPL_START_ADDR fields */
|
|
|
|
#define CMPL_START_ADDR_VALUE(pa) \
|
2017-08-01 18:35:52 +08:00
|
|
|
((u32)((((u64)(pa)) >> RING_CMPL_ALIGN_ORDER) & 0x07ffffff))
|
2017-03-15 14:40:00 +08:00
|
|
|
|
|
|
|
/* Register RING_CONTROL fields */
|
|
|
|
#define CONTROL_MASK_DISABLE_CONTROL 12
|
|
|
|
#define CONTROL_FLUSH_SHIFT 5
|
|
|
|
#define CONTROL_ACTIVE_SHIFT 4
|
|
|
|
#define CONTROL_RATE_ADAPT_MASK 0xf
|
|
|
|
#define CONTROL_RATE_DYNAMIC 0x0
|
|
|
|
#define CONTROL_RATE_FAST 0x8
|
|
|
|
#define CONTROL_RATE_MEDIUM 0x9
|
|
|
|
#define CONTROL_RATE_SLOW 0xa
|
|
|
|
#define CONTROL_RATE_IDLE 0xb
|
|
|
|
|
|
|
|
/* Register RING_FLUSH_DONE fields */
|
|
|
|
#define FLUSH_DONE_MASK 0x1
|
|
|
|
|
|
|
|
/* Register RING_MSI_CONTROL fields */
|
|
|
|
#define MSI_TIMER_VAL_SHIFT 16
|
|
|
|
#define MSI_TIMER_VAL_MASK 0xffff
|
|
|
|
#define MSI_ENABLE_SHIFT 15
|
|
|
|
#define MSI_ENABLE_MASK 0x1
|
|
|
|
#define MSI_COUNT_SHIFT 0
|
|
|
|
#define MSI_COUNT_MASK 0x3ff
|
|
|
|
|
|
|
|
/* Register RING_BD_READ_PTR_DDR_CONTROL fields */
|
|
|
|
#define BD_READ_PTR_DDR_TIMER_VAL_SHIFT 16
|
|
|
|
#define BD_READ_PTR_DDR_TIMER_VAL_MASK 0xffff
|
|
|
|
#define BD_READ_PTR_DDR_ENABLE_SHIFT 15
|
|
|
|
#define BD_READ_PTR_DDR_ENABLE_MASK 0x1
|
|
|
|
|
|
|
|
/* ====== FlexRM ring descriptor defines ===== */
|
|
|
|
|
|
|
|
/* Completion descriptor format */
|
|
|
|
#define CMPL_OPAQUE_SHIFT 0
|
|
|
|
#define CMPL_OPAQUE_MASK 0xffff
|
|
|
|
#define CMPL_ENGINE_STATUS_SHIFT 16
|
|
|
|
#define CMPL_ENGINE_STATUS_MASK 0xffff
|
|
|
|
#define CMPL_DME_STATUS_SHIFT 32
|
|
|
|
#define CMPL_DME_STATUS_MASK 0xffff
|
|
|
|
#define CMPL_RM_STATUS_SHIFT 48
|
|
|
|
#define CMPL_RM_STATUS_MASK 0xffff
|
|
|
|
|
|
|
|
/* Completion DME status code */
|
|
|
|
#define DME_STATUS_MEM_COR_ERR BIT(0)
|
|
|
|
#define DME_STATUS_MEM_UCOR_ERR BIT(1)
|
|
|
|
#define DME_STATUS_FIFO_UNDERFLOW BIT(2)
|
|
|
|
#define DME_STATUS_FIFO_OVERFLOW BIT(3)
|
|
|
|
#define DME_STATUS_RRESP_ERR BIT(4)
|
|
|
|
#define DME_STATUS_BRESP_ERR BIT(5)
|
|
|
|
#define DME_STATUS_ERROR_MASK (DME_STATUS_MEM_COR_ERR | \
|
|
|
|
DME_STATUS_MEM_UCOR_ERR | \
|
|
|
|
DME_STATUS_FIFO_UNDERFLOW | \
|
|
|
|
DME_STATUS_FIFO_OVERFLOW | \
|
|
|
|
DME_STATUS_RRESP_ERR | \
|
|
|
|
DME_STATUS_BRESP_ERR)
|
|
|
|
|
|
|
|
/* Completion RM status code */
|
|
|
|
#define RM_STATUS_CODE_SHIFT 0
|
|
|
|
#define RM_STATUS_CODE_MASK 0x3ff
|
|
|
|
#define RM_STATUS_CODE_GOOD 0x0
|
|
|
|
#define RM_STATUS_CODE_AE_TIMEOUT 0x3ff
|
|
|
|
|
|
|
|
/* General descriptor format */
|
|
|
|
#define DESC_TYPE_SHIFT 60
|
|
|
|
#define DESC_TYPE_MASK 0xf
|
|
|
|
#define DESC_PAYLOAD_SHIFT 0
|
|
|
|
#define DESC_PAYLOAD_MASK 0x0fffffffffffffff
|
|
|
|
|
|
|
|
/* Null descriptor format */
|
|
|
|
#define NULL_TYPE 0
|
|
|
|
#define NULL_TOGGLE_SHIFT 58
|
|
|
|
#define NULL_TOGGLE_MASK 0x1
|
|
|
|
|
|
|
|
/* Header descriptor format */
|
|
|
|
#define HEADER_TYPE 1
|
|
|
|
#define HEADER_TOGGLE_SHIFT 58
|
|
|
|
#define HEADER_TOGGLE_MASK 0x1
|
|
|
|
#define HEADER_ENDPKT_SHIFT 57
|
|
|
|
#define HEADER_ENDPKT_MASK 0x1
|
|
|
|
#define HEADER_STARTPKT_SHIFT 56
|
|
|
|
#define HEADER_STARTPKT_MASK 0x1
|
|
|
|
#define HEADER_BDCOUNT_SHIFT 36
|
|
|
|
#define HEADER_BDCOUNT_MASK 0x1f
|
|
|
|
#define HEADER_BDCOUNT_MAX HEADER_BDCOUNT_MASK
|
|
|
|
#define HEADER_FLAGS_SHIFT 16
|
|
|
|
#define HEADER_FLAGS_MASK 0xffff
|
|
|
|
#define HEADER_OPAQUE_SHIFT 0
|
|
|
|
#define HEADER_OPAQUE_MASK 0xffff
|
|
|
|
|
|
|
|
/* Source (SRC) descriptor format */
|
|
|
|
#define SRC_TYPE 2
|
|
|
|
#define SRC_LENGTH_SHIFT 44
|
|
|
|
#define SRC_LENGTH_MASK 0xffff
|
|
|
|
#define SRC_ADDR_SHIFT 0
|
|
|
|
#define SRC_ADDR_MASK 0x00000fffffffffff
|
|
|
|
|
|
|
|
/* Destination (DST) descriptor format */
|
|
|
|
#define DST_TYPE 3
|
|
|
|
#define DST_LENGTH_SHIFT 44
|
|
|
|
#define DST_LENGTH_MASK 0xffff
|
|
|
|
#define DST_ADDR_SHIFT 0
|
|
|
|
#define DST_ADDR_MASK 0x00000fffffffffff
|
|
|
|
|
|
|
|
/* Immediate (IMM) descriptor format */
|
|
|
|
#define IMM_TYPE 4
|
|
|
|
#define IMM_DATA_SHIFT 0
|
|
|
|
#define IMM_DATA_MASK 0x0fffffffffffffff
|
|
|
|
|
|
|
|
/* Next pointer (NPTR) descriptor format */
|
|
|
|
#define NPTR_TYPE 5
|
|
|
|
#define NPTR_TOGGLE_SHIFT 58
|
|
|
|
#define NPTR_TOGGLE_MASK 0x1
|
|
|
|
#define NPTR_ADDR_SHIFT 0
|
|
|
|
#define NPTR_ADDR_MASK 0x00000fffffffffff
|
|
|
|
|
|
|
|
/* Mega source (MSRC) descriptor format */
|
|
|
|
#define MSRC_TYPE 6
|
|
|
|
#define MSRC_LENGTH_SHIFT 44
|
|
|
|
#define MSRC_LENGTH_MASK 0xffff
|
|
|
|
#define MSRC_ADDR_SHIFT 0
|
|
|
|
#define MSRC_ADDR_MASK 0x00000fffffffffff
|
|
|
|
|
|
|
|
/* Mega destination (MDST) descriptor format */
|
|
|
|
#define MDST_TYPE 7
|
|
|
|
#define MDST_LENGTH_SHIFT 44
|
|
|
|
#define MDST_LENGTH_MASK 0xffff
|
|
|
|
#define MDST_ADDR_SHIFT 0
|
|
|
|
#define MDST_ADDR_MASK 0x00000fffffffffff
|
|
|
|
|
|
|
|
/* Source with tlast (SRCT) descriptor format */
|
|
|
|
#define SRCT_TYPE 8
|
|
|
|
#define SRCT_LENGTH_SHIFT 44
|
|
|
|
#define SRCT_LENGTH_MASK 0xffff
|
|
|
|
#define SRCT_ADDR_SHIFT 0
|
|
|
|
#define SRCT_ADDR_MASK 0x00000fffffffffff
|
|
|
|
|
|
|
|
/* Destination with tlast (DSTT) descriptor format */
|
|
|
|
#define DSTT_TYPE 9
|
|
|
|
#define DSTT_LENGTH_SHIFT 44
|
|
|
|
#define DSTT_LENGTH_MASK 0xffff
|
|
|
|
#define DSTT_ADDR_SHIFT 0
|
|
|
|
#define DSTT_ADDR_MASK 0x00000fffffffffff
|
|
|
|
|
|
|
|
/* Immediate with tlast (IMMT) descriptor format */
|
|
|
|
#define IMMT_TYPE 10
|
|
|
|
#define IMMT_DATA_SHIFT 0
|
|
|
|
#define IMMT_DATA_MASK 0x0fffffffffffffff
|
|
|
|
|
|
|
|
/* Descriptor helper macros */
|
|
|
|
#define DESC_DEC(_d, _s, _m) (((_d) >> (_s)) & (_m))
|
|
|
|
#define DESC_ENC(_d, _v, _s, _m) \
|
|
|
|
do { \
|
|
|
|
(_d) &= ~((u64)(_m) << (_s)); \
|
|
|
|
(_d) |= (((u64)(_v) & (_m)) << (_s)); \
|
|
|
|
} while (0)
|
|
|
|
|
|
|
|
/* ====== FlexRM data structures ===== */
|
|
|
|
|
|
|
|
struct flexrm_ring {
|
|
|
|
/* Unprotected members */
|
|
|
|
int num;
|
|
|
|
struct flexrm_mbox *mbox;
|
|
|
|
void __iomem *regs;
|
|
|
|
bool irq_requested;
|
|
|
|
unsigned int irq;
|
2017-08-01 18:35:50 +08:00
|
|
|
cpumask_t irq_aff_hint;
|
2017-03-15 14:40:00 +08:00
|
|
|
unsigned int msi_timer_val;
|
|
|
|
unsigned int msi_count_threshold;
|
|
|
|
struct brcm_message *requests[RING_MAX_REQ_COUNT];
|
|
|
|
void *bd_base;
|
|
|
|
dma_addr_t bd_dma_base;
|
|
|
|
u32 bd_write_offset;
|
|
|
|
void *cmpl_base;
|
|
|
|
dma_addr_t cmpl_dma_base;
|
2017-08-01 18:35:51 +08:00
|
|
|
/* Atomic stats */
|
|
|
|
atomic_t msg_send_count;
|
|
|
|
atomic_t msg_cmpl_count;
|
2017-03-15 14:40:00 +08:00
|
|
|
/* Protected members */
|
|
|
|
spinlock_t lock;
|
2017-08-01 18:35:53 +08:00
|
|
|
DECLARE_BITMAP(requests_bmap, RING_MAX_REQ_COUNT);
|
2017-03-15 14:40:00 +08:00
|
|
|
u32 cmpl_read_offset;
|
|
|
|
};
|
|
|
|
|
|
|
|
struct flexrm_mbox {
|
|
|
|
struct device *dev;
|
|
|
|
void __iomem *regs;
|
|
|
|
u32 num_rings;
|
|
|
|
struct flexrm_ring *rings;
|
|
|
|
struct dma_pool *bd_pool;
|
|
|
|
struct dma_pool *cmpl_pool;
|
2017-08-01 18:35:51 +08:00
|
|
|
struct dentry *root;
|
|
|
|
struct dentry *config;
|
|
|
|
struct dentry *stats;
|
2017-03-15 14:40:00 +08:00
|
|
|
struct mbox_controller controller;
|
|
|
|
};
|
|
|
|
|
|
|
|
/* ====== FlexRM ring descriptor helper routines ===== */
|
|
|
|
|
|
|
|
static u64 flexrm_read_desc(void *desc_ptr)
|
|
|
|
{
|
|
|
|
return le64_to_cpu(*((u64 *)desc_ptr));
|
|
|
|
}
|
|
|
|
|
|
|
|
static void flexrm_write_desc(void *desc_ptr, u64 desc)
|
|
|
|
{
|
|
|
|
*((u64 *)desc_ptr) = cpu_to_le64(desc);
|
|
|
|
}
|
|
|
|
|
|
|
|
static u32 flexrm_cmpl_desc_to_reqid(u64 cmpl_desc)
|
|
|
|
{
|
|
|
|
return (u32)(cmpl_desc & CMPL_OPAQUE_MASK);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int flexrm_cmpl_desc_to_error(u64 cmpl_desc)
|
|
|
|
{
|
|
|
|
u32 status;
|
|
|
|
|
|
|
|
status = DESC_DEC(cmpl_desc, CMPL_DME_STATUS_SHIFT,
|
|
|
|
CMPL_DME_STATUS_MASK);
|
|
|
|
if (status & DME_STATUS_ERROR_MASK)
|
|
|
|
return -EIO;
|
|
|
|
|
|
|
|
status = DESC_DEC(cmpl_desc, CMPL_RM_STATUS_SHIFT,
|
|
|
|
CMPL_RM_STATUS_MASK);
|
|
|
|
status &= RM_STATUS_CODE_MASK;
|
|
|
|
if (status == RM_STATUS_CODE_AE_TIMEOUT)
|
|
|
|
return -ETIMEDOUT;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool flexrm_is_next_table_desc(void *desc_ptr)
|
|
|
|
{
|
|
|
|
u64 desc = flexrm_read_desc(desc_ptr);
|
|
|
|
u32 type = DESC_DEC(desc, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
|
|
|
|
|
|
|
|
return (type == NPTR_TYPE) ? true : false;
|
|
|
|
}
|
|
|
|
|
|
|
|
static u64 flexrm_next_table_desc(u32 toggle, dma_addr_t next_addr)
|
|
|
|
{
|
|
|
|
u64 desc = 0;
|
|
|
|
|
|
|
|
DESC_ENC(desc, NPTR_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
|
|
|
|
DESC_ENC(desc, toggle, NPTR_TOGGLE_SHIFT, NPTR_TOGGLE_MASK);
|
|
|
|
DESC_ENC(desc, next_addr, NPTR_ADDR_SHIFT, NPTR_ADDR_MASK);
|
|
|
|
|
|
|
|
return desc;
|
|
|
|
}
|
|
|
|
|
|
|
|
static u64 flexrm_null_desc(u32 toggle)
|
|
|
|
{
|
|
|
|
u64 desc = 0;
|
|
|
|
|
|
|
|
DESC_ENC(desc, NULL_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
|
|
|
|
DESC_ENC(desc, toggle, NULL_TOGGLE_SHIFT, NULL_TOGGLE_MASK);
|
|
|
|
|
|
|
|
return desc;
|
|
|
|
}
|
|
|
|
|
|
|
|
static u32 flexrm_estimate_header_desc_count(u32 nhcnt)
|
|
|
|
{
|
|
|
|
u32 hcnt = nhcnt / HEADER_BDCOUNT_MAX;
|
|
|
|
|
|
|
|
if (!(nhcnt % HEADER_BDCOUNT_MAX))
|
|
|
|
hcnt += 1;
|
|
|
|
|
|
|
|
return hcnt;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void flexrm_flip_header_toogle(void *desc_ptr)
|
|
|
|
{
|
|
|
|
u64 desc = flexrm_read_desc(desc_ptr);
|
|
|
|
|
|
|
|
if (desc & ((u64)0x1 << HEADER_TOGGLE_SHIFT))
|
|
|
|
desc &= ~((u64)0x1 << HEADER_TOGGLE_SHIFT);
|
|
|
|
else
|
|
|
|
desc |= ((u64)0x1 << HEADER_TOGGLE_SHIFT);
|
|
|
|
|
|
|
|
flexrm_write_desc(desc_ptr, desc);
|
|
|
|
}
|
|
|
|
|
|
|
|
static u64 flexrm_header_desc(u32 toggle, u32 startpkt, u32 endpkt,
|
|
|
|
u32 bdcount, u32 flags, u32 opaque)
|
|
|
|
{
|
|
|
|
u64 desc = 0;
|
|
|
|
|
|
|
|
DESC_ENC(desc, HEADER_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
|
|
|
|
DESC_ENC(desc, toggle, HEADER_TOGGLE_SHIFT, HEADER_TOGGLE_MASK);
|
|
|
|
DESC_ENC(desc, startpkt, HEADER_STARTPKT_SHIFT, HEADER_STARTPKT_MASK);
|
|
|
|
DESC_ENC(desc, endpkt, HEADER_ENDPKT_SHIFT, HEADER_ENDPKT_MASK);
|
|
|
|
DESC_ENC(desc, bdcount, HEADER_BDCOUNT_SHIFT, HEADER_BDCOUNT_MASK);
|
|
|
|
DESC_ENC(desc, flags, HEADER_FLAGS_SHIFT, HEADER_FLAGS_MASK);
|
|
|
|
DESC_ENC(desc, opaque, HEADER_OPAQUE_SHIFT, HEADER_OPAQUE_MASK);
|
|
|
|
|
|
|
|
return desc;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void flexrm_enqueue_desc(u32 nhpos, u32 nhcnt, u32 reqid,
|
|
|
|
u64 desc, void **desc_ptr, u32 *toggle,
|
|
|
|
void *start_desc, void *end_desc)
|
|
|
|
{
|
|
|
|
u64 d;
|
|
|
|
u32 nhavail, _toggle, _startpkt, _endpkt, _bdcount;
|
|
|
|
|
|
|
|
/* Sanity check */
|
|
|
|
if (nhcnt <= nhpos)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Each request or packet start with a HEADER descriptor followed
|
|
|
|
* by one or more non-HEADER descriptors (SRC, SRCT, MSRC, DST,
|
|
|
|
* DSTT, MDST, IMM, and IMMT). The number of non-HEADER descriptors
|
|
|
|
* following a HEADER descriptor is represented by BDCOUNT field
|
|
|
|
* of HEADER descriptor. The max value of BDCOUNT field is 31 which
|
|
|
|
* means we can only have 31 non-HEADER descriptors following one
|
|
|
|
* HEADER descriptor.
|
|
|
|
*
|
|
|
|
* In general use, number of non-HEADER descriptors can easily go
|
|
|
|
* beyond 31. To tackle this situation, we have packet (or request)
|
|
|
|
* extenstion bits (STARTPKT and ENDPKT) in the HEADER descriptor.
|
|
|
|
*
|
|
|
|
* To use packet extension, the first HEADER descriptor of request
|
|
|
|
* (or packet) will have STARTPKT=1 and ENDPKT=0. The intermediate
|
|
|
|
* HEADER descriptors will have STARTPKT=0 and ENDPKT=0. The last
|
|
|
|
* HEADER descriptor will have STARTPKT=0 and ENDPKT=1. Also, the
|
|
|
|
* TOGGLE bit of the first HEADER will be set to invalid state to
|
|
|
|
* ensure that FlexRM does not start fetching descriptors till all
|
|
|
|
* descriptors are enqueued. The user of this function will flip
|
|
|
|
* the TOGGLE bit of first HEADER after all descriptors are
|
|
|
|
* enqueued.
|
|
|
|
*/
|
|
|
|
|
|
|
|
if ((nhpos % HEADER_BDCOUNT_MAX == 0) && (nhcnt - nhpos)) {
|
|
|
|
/* Prepare the header descriptor */
|
|
|
|
nhavail = (nhcnt - nhpos);
|
|
|
|
_toggle = (nhpos == 0) ? !(*toggle) : (*toggle);
|
|
|
|
_startpkt = (nhpos == 0) ? 0x1 : 0x0;
|
|
|
|
_endpkt = (nhavail <= HEADER_BDCOUNT_MAX) ? 0x1 : 0x0;
|
|
|
|
_bdcount = (nhavail <= HEADER_BDCOUNT_MAX) ?
|
|
|
|
nhavail : HEADER_BDCOUNT_MAX;
|
|
|
|
if (nhavail <= HEADER_BDCOUNT_MAX)
|
|
|
|
_bdcount = nhavail;
|
|
|
|
else
|
|
|
|
_bdcount = HEADER_BDCOUNT_MAX;
|
|
|
|
d = flexrm_header_desc(_toggle, _startpkt, _endpkt,
|
|
|
|
_bdcount, 0x0, reqid);
|
|
|
|
|
|
|
|
/* Write header descriptor */
|
|
|
|
flexrm_write_desc(*desc_ptr, d);
|
|
|
|
|
|
|
|
/* Point to next descriptor */
|
|
|
|
*desc_ptr += sizeof(desc);
|
|
|
|
if (*desc_ptr == end_desc)
|
|
|
|
*desc_ptr = start_desc;
|
|
|
|
|
|
|
|
/* Skip next pointer descriptors */
|
|
|
|
while (flexrm_is_next_table_desc(*desc_ptr)) {
|
|
|
|
*toggle = (*toggle) ? 0 : 1;
|
|
|
|
*desc_ptr += sizeof(desc);
|
|
|
|
if (*desc_ptr == end_desc)
|
|
|
|
*desc_ptr = start_desc;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Write desired descriptor */
|
|
|
|
flexrm_write_desc(*desc_ptr, desc);
|
|
|
|
|
|
|
|
/* Point to next descriptor */
|
|
|
|
*desc_ptr += sizeof(desc);
|
|
|
|
if (*desc_ptr == end_desc)
|
|
|
|
*desc_ptr = start_desc;
|
|
|
|
|
|
|
|
/* Skip next pointer descriptors */
|
|
|
|
while (flexrm_is_next_table_desc(*desc_ptr)) {
|
|
|
|
*toggle = (*toggle) ? 0 : 1;
|
|
|
|
*desc_ptr += sizeof(desc);
|
|
|
|
if (*desc_ptr == end_desc)
|
|
|
|
*desc_ptr = start_desc;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static u64 flexrm_src_desc(dma_addr_t addr, unsigned int length)
|
|
|
|
{
|
|
|
|
u64 desc = 0;
|
|
|
|
|
|
|
|
DESC_ENC(desc, SRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
|
|
|
|
DESC_ENC(desc, length, SRC_LENGTH_SHIFT, SRC_LENGTH_MASK);
|
|
|
|
DESC_ENC(desc, addr, SRC_ADDR_SHIFT, SRC_ADDR_MASK);
|
|
|
|
|
|
|
|
return desc;
|
|
|
|
}
|
|
|
|
|
|
|
|
static u64 flexrm_msrc_desc(dma_addr_t addr, unsigned int length_div_16)
|
|
|
|
{
|
|
|
|
u64 desc = 0;
|
|
|
|
|
|
|
|
DESC_ENC(desc, MSRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
|
|
|
|
DESC_ENC(desc, length_div_16, MSRC_LENGTH_SHIFT, MSRC_LENGTH_MASK);
|
|
|
|
DESC_ENC(desc, addr, MSRC_ADDR_SHIFT, MSRC_ADDR_MASK);
|
|
|
|
|
|
|
|
return desc;
|
|
|
|
}
|
|
|
|
|
|
|
|
static u64 flexrm_dst_desc(dma_addr_t addr, unsigned int length)
|
|
|
|
{
|
|
|
|
u64 desc = 0;
|
|
|
|
|
|
|
|
DESC_ENC(desc, DST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
|
|
|
|
DESC_ENC(desc, length, DST_LENGTH_SHIFT, DST_LENGTH_MASK);
|
|
|
|
DESC_ENC(desc, addr, DST_ADDR_SHIFT, DST_ADDR_MASK);
|
|
|
|
|
|
|
|
return desc;
|
|
|
|
}
|
|
|
|
|
|
|
|
static u64 flexrm_mdst_desc(dma_addr_t addr, unsigned int length_div_16)
|
|
|
|
{
|
|
|
|
u64 desc = 0;
|
|
|
|
|
|
|
|
DESC_ENC(desc, MDST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
|
|
|
|
DESC_ENC(desc, length_div_16, MDST_LENGTH_SHIFT, MDST_LENGTH_MASK);
|
|
|
|
DESC_ENC(desc, addr, MDST_ADDR_SHIFT, MDST_ADDR_MASK);
|
|
|
|
|
|
|
|
return desc;
|
|
|
|
}
|
|
|
|
|
|
|
|
static u64 flexrm_imm_desc(u64 data)
|
|
|
|
{
|
|
|
|
u64 desc = 0;
|
|
|
|
|
|
|
|
DESC_ENC(desc, IMM_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
|
|
|
|
DESC_ENC(desc, data, IMM_DATA_SHIFT, IMM_DATA_MASK);
|
|
|
|
|
|
|
|
return desc;
|
|
|
|
}
|
|
|
|
|
|
|
|
static u64 flexrm_srct_desc(dma_addr_t addr, unsigned int length)
|
|
|
|
{
|
|
|
|
u64 desc = 0;
|
|
|
|
|
|
|
|
DESC_ENC(desc, SRCT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
|
|
|
|
DESC_ENC(desc, length, SRCT_LENGTH_SHIFT, SRCT_LENGTH_MASK);
|
|
|
|
DESC_ENC(desc, addr, SRCT_ADDR_SHIFT, SRCT_ADDR_MASK);
|
|
|
|
|
|
|
|
return desc;
|
|
|
|
}
|
|
|
|
|
|
|
|
static u64 flexrm_dstt_desc(dma_addr_t addr, unsigned int length)
|
|
|
|
{
|
|
|
|
u64 desc = 0;
|
|
|
|
|
|
|
|
DESC_ENC(desc, DSTT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
|
|
|
|
DESC_ENC(desc, length, DSTT_LENGTH_SHIFT, DSTT_LENGTH_MASK);
|
|
|
|
DESC_ENC(desc, addr, DSTT_ADDR_SHIFT, DSTT_ADDR_MASK);
|
|
|
|
|
|
|
|
return desc;
|
|
|
|
}
|
|
|
|
|
|
|
|
static u64 flexrm_immt_desc(u64 data)
|
|
|
|
{
|
|
|
|
u64 desc = 0;
|
|
|
|
|
|
|
|
DESC_ENC(desc, IMMT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
|
|
|
|
DESC_ENC(desc, data, IMMT_DATA_SHIFT, IMMT_DATA_MASK);
|
|
|
|
|
|
|
|
return desc;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool flexrm_spu_sanity_check(struct brcm_message *msg)
|
|
|
|
{
|
|
|
|
struct scatterlist *sg;
|
|
|
|
|
|
|
|
if (!msg->spu.src || !msg->spu.dst)
|
|
|
|
return false;
|
|
|
|
for (sg = msg->spu.src; sg; sg = sg_next(sg)) {
|
|
|
|
if (sg->length & 0xf) {
|
|
|
|
if (sg->length > SRC_LENGTH_MASK)
|
|
|
|
return false;
|
|
|
|
} else {
|
|
|
|
if (sg->length > (MSRC_LENGTH_MASK * 16))
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
for (sg = msg->spu.dst; sg; sg = sg_next(sg)) {
|
|
|
|
if (sg->length & 0xf) {
|
|
|
|
if (sg->length > DST_LENGTH_MASK)
|
|
|
|
return false;
|
|
|
|
} else {
|
|
|
|
if (sg->length > (MDST_LENGTH_MASK * 16))
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
static u32 flexrm_spu_estimate_nonheader_desc_count(struct brcm_message *msg)
|
|
|
|
{
|
|
|
|
u32 cnt = 0;
|
|
|
|
unsigned int dst_target = 0;
|
|
|
|
struct scatterlist *src_sg = msg->spu.src, *dst_sg = msg->spu.dst;
|
|
|
|
|
|
|
|
while (src_sg || dst_sg) {
|
|
|
|
if (src_sg) {
|
|
|
|
cnt++;
|
|
|
|
dst_target = src_sg->length;
|
|
|
|
src_sg = sg_next(src_sg);
|
|
|
|
} else
|
|
|
|
dst_target = UINT_MAX;
|
|
|
|
|
|
|
|
while (dst_target && dst_sg) {
|
|
|
|
cnt++;
|
|
|
|
if (dst_sg->length < dst_target)
|
|
|
|
dst_target -= dst_sg->length;
|
|
|
|
else
|
|
|
|
dst_target = 0;
|
|
|
|
dst_sg = sg_next(dst_sg);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return cnt;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int flexrm_spu_dma_map(struct device *dev, struct brcm_message *msg)
|
|
|
|
{
|
|
|
|
int rc;
|
|
|
|
|
|
|
|
rc = dma_map_sg(dev, msg->spu.src, sg_nents(msg->spu.src),
|
|
|
|
DMA_TO_DEVICE);
|
|
|
|
if (rc < 0)
|
|
|
|
return rc;
|
|
|
|
|
|
|
|
rc = dma_map_sg(dev, msg->spu.dst, sg_nents(msg->spu.dst),
|
|
|
|
DMA_FROM_DEVICE);
|
|
|
|
if (rc < 0) {
|
|
|
|
dma_unmap_sg(dev, msg->spu.src, sg_nents(msg->spu.src),
|
|
|
|
DMA_TO_DEVICE);
|
|
|
|
return rc;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void flexrm_spu_dma_unmap(struct device *dev, struct brcm_message *msg)
|
|
|
|
{
|
|
|
|
dma_unmap_sg(dev, msg->spu.dst, sg_nents(msg->spu.dst),
|
|
|
|
DMA_FROM_DEVICE);
|
|
|
|
dma_unmap_sg(dev, msg->spu.src, sg_nents(msg->spu.src),
|
|
|
|
DMA_TO_DEVICE);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void *flexrm_spu_write_descs(struct brcm_message *msg, u32 nhcnt,
|
|
|
|
u32 reqid, void *desc_ptr, u32 toggle,
|
|
|
|
void *start_desc, void *end_desc)
|
|
|
|
{
|
|
|
|
u64 d;
|
|
|
|
u32 nhpos = 0;
|
|
|
|
void *orig_desc_ptr = desc_ptr;
|
|
|
|
unsigned int dst_target = 0;
|
|
|
|
struct scatterlist *src_sg = msg->spu.src, *dst_sg = msg->spu.dst;
|
|
|
|
|
|
|
|
while (src_sg || dst_sg) {
|
|
|
|
if (src_sg) {
|
|
|
|
if (sg_dma_len(src_sg) & 0xf)
|
|
|
|
d = flexrm_src_desc(sg_dma_address(src_sg),
|
|
|
|
sg_dma_len(src_sg));
|
|
|
|
else
|
|
|
|
d = flexrm_msrc_desc(sg_dma_address(src_sg),
|
|
|
|
sg_dma_len(src_sg)/16);
|
|
|
|
flexrm_enqueue_desc(nhpos, nhcnt, reqid,
|
|
|
|
d, &desc_ptr, &toggle,
|
|
|
|
start_desc, end_desc);
|
|
|
|
nhpos++;
|
|
|
|
dst_target = sg_dma_len(src_sg);
|
|
|
|
src_sg = sg_next(src_sg);
|
|
|
|
} else
|
|
|
|
dst_target = UINT_MAX;
|
|
|
|
|
|
|
|
while (dst_target && dst_sg) {
|
|
|
|
if (sg_dma_len(dst_sg) & 0xf)
|
|
|
|
d = flexrm_dst_desc(sg_dma_address(dst_sg),
|
|
|
|
sg_dma_len(dst_sg));
|
|
|
|
else
|
|
|
|
d = flexrm_mdst_desc(sg_dma_address(dst_sg),
|
|
|
|
sg_dma_len(dst_sg)/16);
|
|
|
|
flexrm_enqueue_desc(nhpos, nhcnt, reqid,
|
|
|
|
d, &desc_ptr, &toggle,
|
|
|
|
start_desc, end_desc);
|
|
|
|
nhpos++;
|
|
|
|
if (sg_dma_len(dst_sg) < dst_target)
|
|
|
|
dst_target -= sg_dma_len(dst_sg);
|
|
|
|
else
|
|
|
|
dst_target = 0;
|
|
|
|
dst_sg = sg_next(dst_sg);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Null descriptor with invalid toggle bit */
|
|
|
|
flexrm_write_desc(desc_ptr, flexrm_null_desc(!toggle));
|
|
|
|
|
|
|
|
/* Ensure that descriptors have been written to memory */
|
|
|
|
wmb();
|
|
|
|
|
|
|
|
/* Flip toggle bit in header */
|
|
|
|
flexrm_flip_header_toogle(orig_desc_ptr);
|
|
|
|
|
|
|
|
return desc_ptr;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool flexrm_sba_sanity_check(struct brcm_message *msg)
|
|
|
|
{
|
|
|
|
u32 i;
|
|
|
|
|
|
|
|
if (!msg->sba.cmds || !msg->sba.cmds_count)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
for (i = 0; i < msg->sba.cmds_count; i++) {
|
|
|
|
if (((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) ||
|
|
|
|
(msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C)) &&
|
|
|
|
(msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT))
|
|
|
|
return false;
|
|
|
|
if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) &&
|
|
|
|
(msg->sba.cmds[i].data_len > SRCT_LENGTH_MASK))
|
|
|
|
return false;
|
|
|
|
if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C) &&
|
|
|
|
(msg->sba.cmds[i].data_len > SRCT_LENGTH_MASK))
|
|
|
|
return false;
|
|
|
|
if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_RESP) &&
|
|
|
|
(msg->sba.cmds[i].resp_len > DSTT_LENGTH_MASK))
|
|
|
|
return false;
|
|
|
|
if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT) &&
|
|
|
|
(msg->sba.cmds[i].data_len > DSTT_LENGTH_MASK))
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
static u32 flexrm_sba_estimate_nonheader_desc_count(struct brcm_message *msg)
|
|
|
|
{
|
|
|
|
u32 i, cnt;
|
|
|
|
|
|
|
|
cnt = 0;
|
|
|
|
for (i = 0; i < msg->sba.cmds_count; i++) {
|
|
|
|
cnt++;
|
|
|
|
|
|
|
|
if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) ||
|
|
|
|
(msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C))
|
|
|
|
cnt++;
|
|
|
|
|
|
|
|
if (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_RESP)
|
|
|
|
cnt++;
|
|
|
|
|
|
|
|
if (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT)
|
|
|
|
cnt++;
|
|
|
|
}
|
|
|
|
|
|
|
|
return cnt;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void *flexrm_sba_write_descs(struct brcm_message *msg, u32 nhcnt,
|
|
|
|
u32 reqid, void *desc_ptr, u32 toggle,
|
|
|
|
void *start_desc, void *end_desc)
|
|
|
|
{
|
|
|
|
u64 d;
|
|
|
|
u32 i, nhpos = 0;
|
|
|
|
struct brcm_sba_command *c;
|
|
|
|
void *orig_desc_ptr = desc_ptr;
|
|
|
|
|
|
|
|
/* Convert SBA commands into descriptors */
|
|
|
|
for (i = 0; i < msg->sba.cmds_count; i++) {
|
|
|
|
c = &msg->sba.cmds[i];
|
|
|
|
|
|
|
|
if ((c->flags & BRCM_SBA_CMD_HAS_RESP) &&
|
|
|
|
(c->flags & BRCM_SBA_CMD_HAS_OUTPUT)) {
|
|
|
|
/* Destination response descriptor */
|
|
|
|
d = flexrm_dst_desc(c->resp, c->resp_len);
|
|
|
|
flexrm_enqueue_desc(nhpos, nhcnt, reqid,
|
|
|
|
d, &desc_ptr, &toggle,
|
|
|
|
start_desc, end_desc);
|
|
|
|
nhpos++;
|
|
|
|
} else if (c->flags & BRCM_SBA_CMD_HAS_RESP) {
|
|
|
|
/* Destination response with tlast descriptor */
|
|
|
|
d = flexrm_dstt_desc(c->resp, c->resp_len);
|
|
|
|
flexrm_enqueue_desc(nhpos, nhcnt, reqid,
|
|
|
|
d, &desc_ptr, &toggle,
|
|
|
|
start_desc, end_desc);
|
|
|
|
nhpos++;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (c->flags & BRCM_SBA_CMD_HAS_OUTPUT) {
|
|
|
|
/* Destination with tlast descriptor */
|
|
|
|
d = flexrm_dstt_desc(c->data, c->data_len);
|
|
|
|
flexrm_enqueue_desc(nhpos, nhcnt, reqid,
|
|
|
|
d, &desc_ptr, &toggle,
|
|
|
|
start_desc, end_desc);
|
|
|
|
nhpos++;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (c->flags & BRCM_SBA_CMD_TYPE_B) {
|
|
|
|
/* Command as immediate descriptor */
|
|
|
|
d = flexrm_imm_desc(c->cmd);
|
|
|
|
flexrm_enqueue_desc(nhpos, nhcnt, reqid,
|
|
|
|
d, &desc_ptr, &toggle,
|
|
|
|
start_desc, end_desc);
|
|
|
|
nhpos++;
|
|
|
|
} else {
|
|
|
|
/* Command as immediate descriptor with tlast */
|
|
|
|
d = flexrm_immt_desc(c->cmd);
|
|
|
|
flexrm_enqueue_desc(nhpos, nhcnt, reqid,
|
|
|
|
d, &desc_ptr, &toggle,
|
|
|
|
start_desc, end_desc);
|
|
|
|
nhpos++;
|
|
|
|
}
|
|
|
|
|
|
|
|
if ((c->flags & BRCM_SBA_CMD_TYPE_B) ||
|
|
|
|
(c->flags & BRCM_SBA_CMD_TYPE_C)) {
|
|
|
|
/* Source with tlast descriptor */
|
|
|
|
d = flexrm_srct_desc(c->data, c->data_len);
|
|
|
|
flexrm_enqueue_desc(nhpos, nhcnt, reqid,
|
|
|
|
d, &desc_ptr, &toggle,
|
|
|
|
start_desc, end_desc);
|
|
|
|
nhpos++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Null descriptor with invalid toggle bit */
|
|
|
|
flexrm_write_desc(desc_ptr, flexrm_null_desc(!toggle));
|
|
|
|
|
|
|
|
/* Ensure that descriptors have been written to memory */
|
|
|
|
wmb();
|
|
|
|
|
|
|
|
/* Flip toggle bit in header */
|
|
|
|
flexrm_flip_header_toogle(orig_desc_ptr);
|
|
|
|
|
|
|
|
return desc_ptr;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool flexrm_sanity_check(struct brcm_message *msg)
|
|
|
|
{
|
|
|
|
if (!msg)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
switch (msg->type) {
|
|
|
|
case BRCM_MESSAGE_SPU:
|
|
|
|
return flexrm_spu_sanity_check(msg);
|
|
|
|
case BRCM_MESSAGE_SBA:
|
|
|
|
return flexrm_sba_sanity_check(msg);
|
|
|
|
default:
|
|
|
|
return false;
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
static u32 flexrm_estimate_nonheader_desc_count(struct brcm_message *msg)
|
|
|
|
{
|
|
|
|
if (!msg)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
switch (msg->type) {
|
|
|
|
case BRCM_MESSAGE_SPU:
|
|
|
|
return flexrm_spu_estimate_nonheader_desc_count(msg);
|
|
|
|
case BRCM_MESSAGE_SBA:
|
|
|
|
return flexrm_sba_estimate_nonheader_desc_count(msg);
|
|
|
|
default:
|
|
|
|
return 0;
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
static int flexrm_dma_map(struct device *dev, struct brcm_message *msg)
|
|
|
|
{
|
|
|
|
if (!dev || !msg)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
switch (msg->type) {
|
|
|
|
case BRCM_MESSAGE_SPU:
|
|
|
|
return flexrm_spu_dma_map(dev, msg);
|
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void flexrm_dma_unmap(struct device *dev, struct brcm_message *msg)
|
|
|
|
{
|
|
|
|
if (!dev || !msg)
|
|
|
|
return;
|
|
|
|
|
|
|
|
switch (msg->type) {
|
|
|
|
case BRCM_MESSAGE_SPU:
|
|
|
|
flexrm_spu_dma_unmap(dev, msg);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void *flexrm_write_descs(struct brcm_message *msg, u32 nhcnt,
|
|
|
|
u32 reqid, void *desc_ptr, u32 toggle,
|
|
|
|
void *start_desc, void *end_desc)
|
|
|
|
{
|
|
|
|
if (!msg || !desc_ptr || !start_desc || !end_desc)
|
|
|
|
return ERR_PTR(-ENOTSUPP);
|
|
|
|
|
|
|
|
if ((desc_ptr < start_desc) || (end_desc <= desc_ptr))
|
|
|
|
return ERR_PTR(-ERANGE);
|
|
|
|
|
|
|
|
switch (msg->type) {
|
|
|
|
case BRCM_MESSAGE_SPU:
|
|
|
|
return flexrm_spu_write_descs(msg, nhcnt, reqid,
|
|
|
|
desc_ptr, toggle,
|
|
|
|
start_desc, end_desc);
|
|
|
|
case BRCM_MESSAGE_SBA:
|
|
|
|
return flexrm_sba_write_descs(msg, nhcnt, reqid,
|
|
|
|
desc_ptr, toggle,
|
|
|
|
start_desc, end_desc);
|
|
|
|
default:
|
|
|
|
return ERR_PTR(-ENOTSUPP);
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
/* ====== FlexRM driver helper routines ===== */
|
|
|
|
|
2017-08-01 18:35:51 +08:00
|
|
|
static void flexrm_write_config_in_seqfile(struct flexrm_mbox *mbox,
|
|
|
|
struct seq_file *file)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
const char *state;
|
|
|
|
struct flexrm_ring *ring;
|
|
|
|
|
|
|
|
seq_printf(file, "%-5s %-9s %-18s %-10s %-18s %-10s\n",
|
|
|
|
"Ring#", "State", "BD_Addr", "BD_Size",
|
|
|
|
"Cmpl_Addr", "Cmpl_Size");
|
|
|
|
|
|
|
|
for (i = 0; i < mbox->num_rings; i++) {
|
|
|
|
ring = &mbox->rings[i];
|
|
|
|
if (readl(ring->regs + RING_CONTROL) &
|
|
|
|
BIT(CONTROL_ACTIVE_SHIFT))
|
|
|
|
state = "active";
|
|
|
|
else
|
|
|
|
state = "inactive";
|
|
|
|
seq_printf(file,
|
|
|
|
"%-5d %-9s 0x%016llx 0x%08x 0x%016llx 0x%08x\n",
|
|
|
|
ring->num, state,
|
|
|
|
(unsigned long long)ring->bd_dma_base,
|
|
|
|
(u32)RING_BD_SIZE,
|
|
|
|
(unsigned long long)ring->cmpl_dma_base,
|
|
|
|
(u32)RING_CMPL_SIZE);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void flexrm_write_stats_in_seqfile(struct flexrm_mbox *mbox,
|
|
|
|
struct seq_file *file)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
u32 val, bd_read_offset;
|
|
|
|
struct flexrm_ring *ring;
|
|
|
|
|
|
|
|
seq_printf(file, "%-5s %-10s %-10s %-10s %-11s %-11s\n",
|
|
|
|
"Ring#", "BD_Read", "BD_Write",
|
|
|
|
"Cmpl_Read", "Submitted", "Completed");
|
|
|
|
|
|
|
|
for (i = 0; i < mbox->num_rings; i++) {
|
|
|
|
ring = &mbox->rings[i];
|
|
|
|
bd_read_offset = readl_relaxed(ring->regs + RING_BD_READ_PTR);
|
|
|
|
val = readl_relaxed(ring->regs + RING_BD_START_ADDR);
|
|
|
|
bd_read_offset *= RING_DESC_SIZE;
|
|
|
|
bd_read_offset += (u32)(BD_START_ADDR_DECODE(val) -
|
|
|
|
ring->bd_dma_base);
|
|
|
|
seq_printf(file, "%-5d 0x%08x 0x%08x 0x%08x %-11d %-11d\n",
|
|
|
|
ring->num,
|
|
|
|
(u32)bd_read_offset,
|
|
|
|
(u32)ring->bd_write_offset,
|
|
|
|
(u32)ring->cmpl_read_offset,
|
|
|
|
(u32)atomic_read(&ring->msg_send_count),
|
|
|
|
(u32)atomic_read(&ring->msg_cmpl_count));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-03-15 14:40:00 +08:00
|
|
|
static int flexrm_new_request(struct flexrm_ring *ring,
|
|
|
|
struct brcm_message *batch_msg,
|
|
|
|
struct brcm_message *msg)
|
|
|
|
{
|
|
|
|
void *next;
|
|
|
|
unsigned long flags;
|
|
|
|
u32 val, count, nhcnt;
|
|
|
|
u32 read_offset, write_offset;
|
|
|
|
bool exit_cleanup = false;
|
|
|
|
int ret = 0, reqid;
|
|
|
|
|
|
|
|
/* Do sanity check on message */
|
|
|
|
if (!flexrm_sanity_check(msg))
|
|
|
|
return -EIO;
|
|
|
|
msg->error = 0;
|
|
|
|
|
|
|
|
/* If no requests possible then save data pointer and goto done. */
|
2017-08-01 18:35:53 +08:00
|
|
|
spin_lock_irqsave(&ring->lock, flags);
|
|
|
|
reqid = bitmap_find_free_region(ring->requests_bmap,
|
|
|
|
RING_MAX_REQ_COUNT, 0);
|
|
|
|
spin_unlock_irqrestore(&ring->lock, flags);
|
2017-08-01 18:35:54 +08:00
|
|
|
if (reqid < 0)
|
|
|
|
return -ENOSPC;
|
2017-03-15 14:40:00 +08:00
|
|
|
ring->requests[reqid] = msg;
|
|
|
|
|
|
|
|
/* Do DMA mappings for the message */
|
|
|
|
ret = flexrm_dma_map(ring->mbox->dev, msg);
|
|
|
|
if (ret < 0) {
|
|
|
|
ring->requests[reqid] = NULL;
|
2017-08-01 18:35:53 +08:00
|
|
|
spin_lock_irqsave(&ring->lock, flags);
|
|
|
|
bitmap_release_region(ring->requests_bmap, reqid, 0);
|
|
|
|
spin_unlock_irqrestore(&ring->lock, flags);
|
2017-03-15 14:40:00 +08:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Determine current HW BD read offset */
|
|
|
|
read_offset = readl_relaxed(ring->regs + RING_BD_READ_PTR);
|
|
|
|
val = readl_relaxed(ring->regs + RING_BD_START_ADDR);
|
|
|
|
read_offset *= RING_DESC_SIZE;
|
|
|
|
read_offset += (u32)(BD_START_ADDR_DECODE(val) - ring->bd_dma_base);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Number required descriptors = number of non-header descriptors +
|
|
|
|
* number of header descriptors +
|
|
|
|
* 1x null descriptor
|
|
|
|
*/
|
|
|
|
nhcnt = flexrm_estimate_nonheader_desc_count(msg);
|
|
|
|
count = flexrm_estimate_header_desc_count(nhcnt) + nhcnt + 1;
|
|
|
|
|
|
|
|
/* Check for available descriptor space. */
|
|
|
|
write_offset = ring->bd_write_offset;
|
|
|
|
while (count) {
|
|
|
|
if (!flexrm_is_next_table_desc(ring->bd_base + write_offset))
|
|
|
|
count--;
|
|
|
|
write_offset += RING_DESC_SIZE;
|
|
|
|
if (write_offset == RING_BD_SIZE)
|
|
|
|
write_offset = 0x0;
|
|
|
|
if (write_offset == read_offset)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (count) {
|
2017-08-01 18:35:54 +08:00
|
|
|
ret = -ENOSPC;
|
2017-03-15 14:40:00 +08:00
|
|
|
exit_cleanup = true;
|
|
|
|
goto exit;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Write descriptors to ring */
|
|
|
|
next = flexrm_write_descs(msg, nhcnt, reqid,
|
|
|
|
ring->bd_base + ring->bd_write_offset,
|
|
|
|
RING_BD_TOGGLE_VALID(ring->bd_write_offset),
|
|
|
|
ring->bd_base, ring->bd_base + RING_BD_SIZE);
|
|
|
|
if (IS_ERR(next)) {
|
|
|
|
ret = PTR_ERR(next);
|
|
|
|
exit_cleanup = true;
|
|
|
|
goto exit;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Save ring BD write offset */
|
|
|
|
ring->bd_write_offset = (unsigned long)(next - ring->bd_base);
|
|
|
|
|
2017-08-01 18:35:51 +08:00
|
|
|
/* Increment number of messages sent */
|
|
|
|
atomic_inc_return(&ring->msg_send_count);
|
|
|
|
|
2017-03-15 14:40:00 +08:00
|
|
|
exit:
|
|
|
|
/* Update error status in message */
|
|
|
|
msg->error = ret;
|
|
|
|
|
|
|
|
/* Cleanup if we failed */
|
|
|
|
if (exit_cleanup) {
|
|
|
|
flexrm_dma_unmap(ring->mbox->dev, msg);
|
|
|
|
ring->requests[reqid] = NULL;
|
2017-08-01 18:35:53 +08:00
|
|
|
spin_lock_irqsave(&ring->lock, flags);
|
|
|
|
bitmap_release_region(ring->requests_bmap, reqid, 0);
|
|
|
|
spin_unlock_irqrestore(&ring->lock, flags);
|
2017-03-15 14:40:00 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int flexrm_process_completions(struct flexrm_ring *ring)
|
|
|
|
{
|
|
|
|
u64 desc;
|
|
|
|
int err, count = 0;
|
|
|
|
unsigned long flags;
|
|
|
|
struct brcm_message *msg = NULL;
|
|
|
|
u32 reqid, cmpl_read_offset, cmpl_write_offset;
|
|
|
|
struct mbox_chan *chan = &ring->mbox->controller.chans[ring->num];
|
|
|
|
|
|
|
|
spin_lock_irqsave(&ring->lock, flags);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Get current completion read and write offset
|
|
|
|
*
|
|
|
|
* Note: We should read completion write pointer atleast once
|
|
|
|
* after we get a MSI interrupt because HW maintains internal
|
|
|
|
* MSI status which will allow next MSI interrupt only after
|
|
|
|
* completion write pointer is read.
|
|
|
|
*/
|
|
|
|
cmpl_write_offset = readl_relaxed(ring->regs + RING_CMPL_WRITE_PTR);
|
|
|
|
cmpl_write_offset *= RING_DESC_SIZE;
|
|
|
|
cmpl_read_offset = ring->cmpl_read_offset;
|
|
|
|
ring->cmpl_read_offset = cmpl_write_offset;
|
|
|
|
|
|
|
|
spin_unlock_irqrestore(&ring->lock, flags);
|
|
|
|
|
|
|
|
/* For each completed request notify mailbox clients */
|
|
|
|
reqid = 0;
|
|
|
|
while (cmpl_read_offset != cmpl_write_offset) {
|
|
|
|
/* Dequeue next completion descriptor */
|
|
|
|
desc = *((u64 *)(ring->cmpl_base + cmpl_read_offset));
|
|
|
|
|
|
|
|
/* Next read offset */
|
|
|
|
cmpl_read_offset += RING_DESC_SIZE;
|
|
|
|
if (cmpl_read_offset == RING_CMPL_SIZE)
|
|
|
|
cmpl_read_offset = 0;
|
|
|
|
|
|
|
|
/* Decode error from completion descriptor */
|
|
|
|
err = flexrm_cmpl_desc_to_error(desc);
|
|
|
|
if (err < 0) {
|
|
|
|
dev_warn(ring->mbox->dev,
|
|
|
|
"got completion desc=0x%lx with error %d",
|
|
|
|
(unsigned long)desc, err);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Determine request id from completion descriptor */
|
|
|
|
reqid = flexrm_cmpl_desc_to_reqid(desc);
|
|
|
|
|
|
|
|
/* Determine message pointer based on reqid */
|
|
|
|
msg = ring->requests[reqid];
|
|
|
|
if (!msg) {
|
|
|
|
dev_warn(ring->mbox->dev,
|
|
|
|
"null msg pointer for completion desc=0x%lx",
|
|
|
|
(unsigned long)desc);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Release reqid for recycling */
|
|
|
|
ring->requests[reqid] = NULL;
|
2017-08-01 18:35:53 +08:00
|
|
|
spin_lock_irqsave(&ring->lock, flags);
|
|
|
|
bitmap_release_region(ring->requests_bmap, reqid, 0);
|
|
|
|
spin_unlock_irqrestore(&ring->lock, flags);
|
2017-03-15 14:40:00 +08:00
|
|
|
|
|
|
|
/* Unmap DMA mappings */
|
|
|
|
flexrm_dma_unmap(ring->mbox->dev, msg);
|
|
|
|
|
|
|
|
/* Give-back message to mailbox client */
|
|
|
|
msg->error = err;
|
|
|
|
mbox_chan_received_data(chan, msg);
|
|
|
|
|
|
|
|
/* Increment number of completions processed */
|
2017-08-01 18:35:51 +08:00
|
|
|
atomic_inc_return(&ring->msg_cmpl_count);
|
2017-03-15 14:40:00 +08:00
|
|
|
count++;
|
|
|
|
}
|
|
|
|
|
|
|
|
return count;
|
|
|
|
}
|
|
|
|
|
2017-08-01 18:35:51 +08:00
|
|
|
/* ====== FlexRM Debugfs callbacks ====== */
|
|
|
|
|
|
|
|
static int flexrm_debugfs_conf_show(struct seq_file *file, void *offset)
|
|
|
|
{
|
|
|
|
struct platform_device *pdev = to_platform_device(file->private);
|
|
|
|
struct flexrm_mbox *mbox = platform_get_drvdata(pdev);
|
|
|
|
|
|
|
|
/* Write config in file */
|
|
|
|
flexrm_write_config_in_seqfile(mbox, file);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int flexrm_debugfs_stats_show(struct seq_file *file, void *offset)
|
|
|
|
{
|
|
|
|
struct platform_device *pdev = to_platform_device(file->private);
|
|
|
|
struct flexrm_mbox *mbox = platform_get_drvdata(pdev);
|
|
|
|
|
|
|
|
/* Write stats in file */
|
|
|
|
flexrm_write_stats_in_seqfile(mbox, file);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-03-15 14:40:00 +08:00
|
|
|
/* ====== FlexRM interrupt handler ===== */
|
|
|
|
|
|
|
|
static irqreturn_t flexrm_irq_event(int irq, void *dev_id)
|
|
|
|
{
|
|
|
|
/* We only have MSI for completions so just wakeup IRQ thread */
|
|
|
|
/* Ring related errors will be informed via completion descriptors */
|
|
|
|
|
|
|
|
return IRQ_WAKE_THREAD;
|
|
|
|
}
|
|
|
|
|
|
|
|
static irqreturn_t flexrm_irq_thread(int irq, void *dev_id)
|
|
|
|
{
|
|
|
|
flexrm_process_completions(dev_id);
|
|
|
|
|
|
|
|
return IRQ_HANDLED;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* ====== FlexRM mailbox callbacks ===== */
|
|
|
|
|
|
|
|
static int flexrm_send_data(struct mbox_chan *chan, void *data)
|
|
|
|
{
|
|
|
|
int i, rc;
|
|
|
|
struct flexrm_ring *ring = chan->con_priv;
|
|
|
|
struct brcm_message *msg = data;
|
|
|
|
|
|
|
|
if (msg->type == BRCM_MESSAGE_BATCH) {
|
|
|
|
for (i = msg->batch.msgs_queued;
|
|
|
|
i < msg->batch.msgs_count; i++) {
|
|
|
|
rc = flexrm_new_request(ring, msg,
|
|
|
|
&msg->batch.msgs[i]);
|
|
|
|
if (rc) {
|
|
|
|
msg->error = rc;
|
|
|
|
return rc;
|
|
|
|
}
|
|
|
|
msg->batch.msgs_queued++;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
return flexrm_new_request(ring, NULL, data);
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool flexrm_peek_data(struct mbox_chan *chan)
|
|
|
|
{
|
|
|
|
int cnt = flexrm_process_completions(chan->con_priv);
|
|
|
|
|
|
|
|
return (cnt > 0) ? true : false;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int flexrm_startup(struct mbox_chan *chan)
|
|
|
|
{
|
|
|
|
u64 d;
|
|
|
|
u32 val, off;
|
|
|
|
int ret = 0;
|
|
|
|
dma_addr_t next_addr;
|
|
|
|
struct flexrm_ring *ring = chan->con_priv;
|
|
|
|
|
|
|
|
/* Allocate BD memory */
|
|
|
|
ring->bd_base = dma_pool_alloc(ring->mbox->bd_pool,
|
|
|
|
GFP_KERNEL, &ring->bd_dma_base);
|
|
|
|
if (!ring->bd_base) {
|
|
|
|
dev_err(ring->mbox->dev, "can't allocate BD memory\n");
|
|
|
|
ret = -ENOMEM;
|
|
|
|
goto fail;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Configure next table pointer entries in BD memory */
|
|
|
|
for (off = 0; off < RING_BD_SIZE; off += RING_DESC_SIZE) {
|
|
|
|
next_addr = off + RING_DESC_SIZE;
|
|
|
|
if (next_addr == RING_BD_SIZE)
|
|
|
|
next_addr = 0;
|
|
|
|
next_addr += ring->bd_dma_base;
|
|
|
|
if (RING_BD_ALIGN_CHECK(next_addr))
|
|
|
|
d = flexrm_next_table_desc(RING_BD_TOGGLE_VALID(off),
|
|
|
|
next_addr);
|
|
|
|
else
|
|
|
|
d = flexrm_null_desc(RING_BD_TOGGLE_INVALID(off));
|
|
|
|
flexrm_write_desc(ring->bd_base + off, d);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Allocate completion memory */
|
|
|
|
ring->cmpl_base = dma_pool_alloc(ring->mbox->cmpl_pool,
|
|
|
|
GFP_KERNEL, &ring->cmpl_dma_base);
|
|
|
|
if (!ring->cmpl_base) {
|
|
|
|
dev_err(ring->mbox->dev, "can't allocate completion memory\n");
|
|
|
|
ret = -ENOMEM;
|
|
|
|
goto fail_free_bd_memory;
|
|
|
|
}
|
|
|
|
memset(ring->cmpl_base, 0, RING_CMPL_SIZE);
|
|
|
|
|
|
|
|
/* Request IRQ */
|
|
|
|
if (ring->irq == UINT_MAX) {
|
|
|
|
dev_err(ring->mbox->dev, "ring IRQ not available\n");
|
|
|
|
ret = -ENODEV;
|
|
|
|
goto fail_free_cmpl_memory;
|
|
|
|
}
|
|
|
|
ret = request_threaded_irq(ring->irq,
|
|
|
|
flexrm_irq_event,
|
|
|
|
flexrm_irq_thread,
|
|
|
|
0, dev_name(ring->mbox->dev), ring);
|
|
|
|
if (ret) {
|
|
|
|
dev_err(ring->mbox->dev, "failed to request ring IRQ\n");
|
|
|
|
goto fail_free_cmpl_memory;
|
|
|
|
}
|
|
|
|
ring->irq_requested = true;
|
|
|
|
|
2017-08-01 18:35:50 +08:00
|
|
|
/* Set IRQ affinity hint */
|
|
|
|
ring->irq_aff_hint = CPU_MASK_NONE;
|
|
|
|
val = ring->mbox->num_rings;
|
|
|
|
val = (num_online_cpus() < val) ? val / num_online_cpus() : 1;
|
|
|
|
cpumask_set_cpu((ring->num / val) % num_online_cpus(),
|
|
|
|
&ring->irq_aff_hint);
|
|
|
|
ret = irq_set_affinity_hint(ring->irq, &ring->irq_aff_hint);
|
|
|
|
if (ret) {
|
|
|
|
dev_err(ring->mbox->dev, "failed to set IRQ affinity hint\n");
|
|
|
|
goto fail_free_irq;
|
|
|
|
}
|
|
|
|
|
2017-03-15 14:40:00 +08:00
|
|
|
/* Disable/inactivate ring */
|
|
|
|
writel_relaxed(0x0, ring->regs + RING_CONTROL);
|
|
|
|
|
|
|
|
/* Program BD start address */
|
|
|
|
val = BD_START_ADDR_VALUE(ring->bd_dma_base);
|
|
|
|
writel_relaxed(val, ring->regs + RING_BD_START_ADDR);
|
|
|
|
|
|
|
|
/* BD write pointer will be same as HW write pointer */
|
|
|
|
ring->bd_write_offset =
|
|
|
|
readl_relaxed(ring->regs + RING_BD_WRITE_PTR);
|
|
|
|
ring->bd_write_offset *= RING_DESC_SIZE;
|
|
|
|
|
|
|
|
/* Program completion start address */
|
|
|
|
val = CMPL_START_ADDR_VALUE(ring->cmpl_dma_base);
|
|
|
|
writel_relaxed(val, ring->regs + RING_CMPL_START_ADDR);
|
|
|
|
|
|
|
|
/* Completion read pointer will be same as HW write pointer */
|
|
|
|
ring->cmpl_read_offset =
|
|
|
|
readl_relaxed(ring->regs + RING_CMPL_WRITE_PTR);
|
|
|
|
ring->cmpl_read_offset *= RING_DESC_SIZE;
|
|
|
|
|
|
|
|
/* Read ring Tx, Rx, and Outstanding counts to clear */
|
|
|
|
readl_relaxed(ring->regs + RING_NUM_REQ_RECV_LS);
|
|
|
|
readl_relaxed(ring->regs + RING_NUM_REQ_RECV_MS);
|
|
|
|
readl_relaxed(ring->regs + RING_NUM_REQ_TRANS_LS);
|
|
|
|
readl_relaxed(ring->regs + RING_NUM_REQ_TRANS_MS);
|
|
|
|
readl_relaxed(ring->regs + RING_NUM_REQ_OUTSTAND);
|
|
|
|
|
|
|
|
/* Configure RING_MSI_CONTROL */
|
|
|
|
val = 0;
|
|
|
|
val |= (ring->msi_timer_val << MSI_TIMER_VAL_SHIFT);
|
|
|
|
val |= BIT(MSI_ENABLE_SHIFT);
|
|
|
|
val |= (ring->msi_count_threshold & MSI_COUNT_MASK) << MSI_COUNT_SHIFT;
|
|
|
|
writel_relaxed(val, ring->regs + RING_MSI_CONTROL);
|
|
|
|
|
|
|
|
/* Enable/activate ring */
|
|
|
|
val = BIT(CONTROL_ACTIVE_SHIFT);
|
|
|
|
writel_relaxed(val, ring->regs + RING_CONTROL);
|
|
|
|
|
2017-08-01 18:35:51 +08:00
|
|
|
/* Reset stats to zero */
|
|
|
|
atomic_set(&ring->msg_send_count, 0);
|
|
|
|
atomic_set(&ring->msg_cmpl_count, 0);
|
|
|
|
|
2017-03-15 14:40:00 +08:00
|
|
|
return 0;
|
|
|
|
|
2017-08-01 18:35:50 +08:00
|
|
|
fail_free_irq:
|
|
|
|
free_irq(ring->irq, ring);
|
|
|
|
ring->irq_requested = false;
|
2017-03-15 14:40:00 +08:00
|
|
|
fail_free_cmpl_memory:
|
|
|
|
dma_pool_free(ring->mbox->cmpl_pool,
|
|
|
|
ring->cmpl_base, ring->cmpl_dma_base);
|
|
|
|
ring->cmpl_base = NULL;
|
|
|
|
fail_free_bd_memory:
|
|
|
|
dma_pool_free(ring->mbox->bd_pool,
|
|
|
|
ring->bd_base, ring->bd_dma_base);
|
|
|
|
ring->bd_base = NULL;
|
|
|
|
fail:
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void flexrm_shutdown(struct mbox_chan *chan)
|
|
|
|
{
|
|
|
|
u32 reqid;
|
|
|
|
unsigned int timeout;
|
|
|
|
struct brcm_message *msg;
|
|
|
|
struct flexrm_ring *ring = chan->con_priv;
|
|
|
|
|
|
|
|
/* Disable/inactivate ring */
|
|
|
|
writel_relaxed(0x0, ring->regs + RING_CONTROL);
|
|
|
|
|
|
|
|
/* Flush ring with timeout of 1s */
|
|
|
|
timeout = 1000;
|
|
|
|
writel_relaxed(BIT(CONTROL_FLUSH_SHIFT),
|
|
|
|
ring->regs + RING_CONTROL);
|
|
|
|
do {
|
|
|
|
if (readl_relaxed(ring->regs + RING_FLUSH_DONE) &
|
|
|
|
FLUSH_DONE_MASK)
|
|
|
|
break;
|
|
|
|
mdelay(1);
|
|
|
|
} while (timeout--);
|
|
|
|
|
|
|
|
/* Abort all in-flight requests */
|
|
|
|
for (reqid = 0; reqid < RING_MAX_REQ_COUNT; reqid++) {
|
|
|
|
msg = ring->requests[reqid];
|
|
|
|
if (!msg)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/* Release reqid for recycling */
|
|
|
|
ring->requests[reqid] = NULL;
|
|
|
|
|
|
|
|
/* Unmap DMA mappings */
|
|
|
|
flexrm_dma_unmap(ring->mbox->dev, msg);
|
|
|
|
|
|
|
|
/* Give-back message to mailbox client */
|
|
|
|
msg->error = -EIO;
|
|
|
|
mbox_chan_received_data(chan, msg);
|
|
|
|
}
|
|
|
|
|
2017-08-01 18:35:53 +08:00
|
|
|
/* Clear requests bitmap */
|
|
|
|
bitmap_zero(ring->requests_bmap, RING_MAX_REQ_COUNT);
|
|
|
|
|
2017-03-15 14:40:00 +08:00
|
|
|
/* Release IRQ */
|
|
|
|
if (ring->irq_requested) {
|
2017-08-01 18:35:50 +08:00
|
|
|
irq_set_affinity_hint(ring->irq, NULL);
|
2017-03-15 14:40:00 +08:00
|
|
|
free_irq(ring->irq, ring);
|
|
|
|
ring->irq_requested = false;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Free-up completion descriptor ring */
|
|
|
|
if (ring->cmpl_base) {
|
|
|
|
dma_pool_free(ring->mbox->cmpl_pool,
|
|
|
|
ring->cmpl_base, ring->cmpl_dma_base);
|
|
|
|
ring->cmpl_base = NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Free-up BD descriptor ring */
|
|
|
|
if (ring->bd_base) {
|
|
|
|
dma_pool_free(ring->mbox->bd_pool,
|
|
|
|
ring->bd_base, ring->bd_dma_base);
|
|
|
|
ring->bd_base = NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static const struct mbox_chan_ops flexrm_mbox_chan_ops = {
|
|
|
|
.send_data = flexrm_send_data,
|
|
|
|
.startup = flexrm_startup,
|
|
|
|
.shutdown = flexrm_shutdown,
|
|
|
|
.peek_data = flexrm_peek_data,
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct mbox_chan *flexrm_mbox_of_xlate(struct mbox_controller *cntlr,
|
|
|
|
const struct of_phandle_args *pa)
|
|
|
|
{
|
|
|
|
struct mbox_chan *chan;
|
|
|
|
struct flexrm_ring *ring;
|
|
|
|
|
|
|
|
if (pa->args_count < 3)
|
|
|
|
return ERR_PTR(-EINVAL);
|
|
|
|
|
|
|
|
if (pa->args[0] >= cntlr->num_chans)
|
|
|
|
return ERR_PTR(-ENOENT);
|
|
|
|
|
|
|
|
if (pa->args[1] > MSI_COUNT_MASK)
|
|
|
|
return ERR_PTR(-EINVAL);
|
|
|
|
|
|
|
|
if (pa->args[2] > MSI_TIMER_VAL_MASK)
|
|
|
|
return ERR_PTR(-EINVAL);
|
|
|
|
|
|
|
|
chan = &cntlr->chans[pa->args[0]];
|
|
|
|
ring = chan->con_priv;
|
|
|
|
ring->msi_count_threshold = pa->args[1];
|
|
|
|
ring->msi_timer_val = pa->args[2];
|
|
|
|
|
|
|
|
return chan;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* ====== FlexRM platform driver ===== */
|
|
|
|
|
|
|
|
static void flexrm_mbox_msi_write(struct msi_desc *desc, struct msi_msg *msg)
|
|
|
|
{
|
|
|
|
struct device *dev = msi_desc_to_dev(desc);
|
|
|
|
struct flexrm_mbox *mbox = dev_get_drvdata(dev);
|
|
|
|
struct flexrm_ring *ring = &mbox->rings[desc->platform.msi_index];
|
|
|
|
|
|
|
|
/* Configure per-Ring MSI registers */
|
|
|
|
writel_relaxed(msg->address_lo, ring->regs + RING_MSI_ADDR_LS);
|
|
|
|
writel_relaxed(msg->address_hi, ring->regs + RING_MSI_ADDR_MS);
|
|
|
|
writel_relaxed(msg->data, ring->regs + RING_MSI_DATA_VALUE);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int flexrm_mbox_probe(struct platform_device *pdev)
|
|
|
|
{
|
|
|
|
int index, ret = 0;
|
|
|
|
void __iomem *regs;
|
|
|
|
void __iomem *regs_end;
|
|
|
|
struct msi_desc *desc;
|
|
|
|
struct resource *iomem;
|
|
|
|
struct flexrm_ring *ring;
|
|
|
|
struct flexrm_mbox *mbox;
|
|
|
|
struct device *dev = &pdev->dev;
|
|
|
|
|
|
|
|
/* Allocate driver mailbox struct */
|
|
|
|
mbox = devm_kzalloc(dev, sizeof(*mbox), GFP_KERNEL);
|
|
|
|
if (!mbox) {
|
|
|
|
ret = -ENOMEM;
|
|
|
|
goto fail;
|
|
|
|
}
|
|
|
|
mbox->dev = dev;
|
|
|
|
platform_set_drvdata(pdev, mbox);
|
|
|
|
|
|
|
|
/* Get resource for registers */
|
|
|
|
iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
|
|
|
|
if (!iomem || (resource_size(iomem) < RING_REGS_SIZE)) {
|
|
|
|
ret = -ENODEV;
|
|
|
|
goto fail;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Map registers of all rings */
|
|
|
|
mbox->regs = devm_ioremap_resource(&pdev->dev, iomem);
|
|
|
|
if (IS_ERR(mbox->regs)) {
|
|
|
|
ret = PTR_ERR(mbox->regs);
|
|
|
|
dev_err(&pdev->dev, "Failed to remap mailbox regs: %d\n", ret);
|
|
|
|
goto fail;
|
|
|
|
}
|
|
|
|
regs_end = mbox->regs + resource_size(iomem);
|
|
|
|
|
|
|
|
/* Scan and count available rings */
|
|
|
|
mbox->num_rings = 0;
|
|
|
|
for (regs = mbox->regs; regs < regs_end; regs += RING_REGS_SIZE) {
|
|
|
|
if (readl_relaxed(regs + RING_VER) == RING_VER_MAGIC)
|
|
|
|
mbox->num_rings++;
|
|
|
|
}
|
|
|
|
if (!mbox->num_rings) {
|
|
|
|
ret = -ENODEV;
|
|
|
|
goto fail;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Allocate driver ring structs */
|
|
|
|
ring = devm_kcalloc(dev, mbox->num_rings, sizeof(*ring), GFP_KERNEL);
|
|
|
|
if (!ring) {
|
|
|
|
ret = -ENOMEM;
|
|
|
|
goto fail;
|
|
|
|
}
|
|
|
|
mbox->rings = ring;
|
|
|
|
|
|
|
|
/* Initialize members of driver ring structs */
|
|
|
|
regs = mbox->regs;
|
|
|
|
for (index = 0; index < mbox->num_rings; index++) {
|
|
|
|
ring = &mbox->rings[index];
|
|
|
|
ring->num = index;
|
|
|
|
ring->mbox = mbox;
|
|
|
|
while ((regs < regs_end) &&
|
|
|
|
(readl_relaxed(regs + RING_VER) != RING_VER_MAGIC))
|
|
|
|
regs += RING_REGS_SIZE;
|
|
|
|
if (regs_end <= regs) {
|
|
|
|
ret = -ENODEV;
|
|
|
|
goto fail;
|
|
|
|
}
|
|
|
|
ring->regs = regs;
|
|
|
|
regs += RING_REGS_SIZE;
|
|
|
|
ring->irq = UINT_MAX;
|
|
|
|
ring->irq_requested = false;
|
|
|
|
ring->msi_timer_val = MSI_TIMER_VAL_MASK;
|
|
|
|
ring->msi_count_threshold = 0x1;
|
|
|
|
memset(ring->requests, 0, sizeof(ring->requests));
|
|
|
|
ring->bd_base = NULL;
|
|
|
|
ring->bd_dma_base = 0;
|
|
|
|
ring->cmpl_base = NULL;
|
|
|
|
ring->cmpl_dma_base = 0;
|
2017-08-01 18:35:51 +08:00
|
|
|
atomic_set(&ring->msg_send_count, 0);
|
|
|
|
atomic_set(&ring->msg_cmpl_count, 0);
|
2017-03-15 14:40:00 +08:00
|
|
|
spin_lock_init(&ring->lock);
|
2017-08-01 18:35:53 +08:00
|
|
|
bitmap_zero(ring->requests_bmap, RING_MAX_REQ_COUNT);
|
2017-03-15 14:40:00 +08:00
|
|
|
ring->cmpl_read_offset = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* FlexRM is capable of 40-bit physical addresses only */
|
|
|
|
ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40));
|
|
|
|
if (ret) {
|
|
|
|
ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
|
|
|
|
if (ret)
|
|
|
|
goto fail;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Create DMA pool for ring BD memory */
|
|
|
|
mbox->bd_pool = dma_pool_create("bd", dev, RING_BD_SIZE,
|
|
|
|
1 << RING_BD_ALIGN_ORDER, 0);
|
|
|
|
if (!mbox->bd_pool) {
|
|
|
|
ret = -ENOMEM;
|
|
|
|
goto fail;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Create DMA pool for ring completion memory */
|
|
|
|
mbox->cmpl_pool = dma_pool_create("cmpl", dev, RING_CMPL_SIZE,
|
|
|
|
1 << RING_CMPL_ALIGN_ORDER, 0);
|
|
|
|
if (!mbox->cmpl_pool) {
|
|
|
|
ret = -ENOMEM;
|
|
|
|
goto fail_destroy_bd_pool;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Allocate platform MSIs for each ring */
|
|
|
|
ret = platform_msi_domain_alloc_irqs(dev, mbox->num_rings,
|
|
|
|
flexrm_mbox_msi_write);
|
|
|
|
if (ret)
|
|
|
|
goto fail_destroy_cmpl_pool;
|
|
|
|
|
|
|
|
/* Save alloced IRQ numbers for each ring */
|
|
|
|
for_each_msi_entry(desc, dev) {
|
|
|
|
ring = &mbox->rings[desc->platform.msi_index];
|
|
|
|
ring->irq = desc->irq;
|
|
|
|
}
|
|
|
|
|
2017-08-01 18:35:51 +08:00
|
|
|
/* Check availability of debugfs */
|
|
|
|
if (!debugfs_initialized())
|
|
|
|
goto skip_debugfs;
|
|
|
|
|
|
|
|
/* Create debugfs root entry */
|
|
|
|
mbox->root = debugfs_create_dir(dev_name(mbox->dev), NULL);
|
|
|
|
if (IS_ERR_OR_NULL(mbox->root)) {
|
|
|
|
ret = PTR_ERR_OR_ZERO(mbox->root);
|
|
|
|
goto fail_free_msis;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Create debugfs config entry */
|
|
|
|
mbox->config = debugfs_create_devm_seqfile(mbox->dev,
|
|
|
|
"config", mbox->root,
|
|
|
|
flexrm_debugfs_conf_show);
|
|
|
|
if (IS_ERR_OR_NULL(mbox->config)) {
|
|
|
|
ret = PTR_ERR_OR_ZERO(mbox->config);
|
|
|
|
goto fail_free_debugfs_root;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Create debugfs stats entry */
|
|
|
|
mbox->stats = debugfs_create_devm_seqfile(mbox->dev,
|
|
|
|
"stats", mbox->root,
|
|
|
|
flexrm_debugfs_stats_show);
|
|
|
|
if (IS_ERR_OR_NULL(mbox->stats)) {
|
|
|
|
ret = PTR_ERR_OR_ZERO(mbox->stats);
|
|
|
|
goto fail_free_debugfs_root;
|
|
|
|
}
|
|
|
|
skip_debugfs:
|
|
|
|
|
2017-03-15 14:40:00 +08:00
|
|
|
/* Initialize mailbox controller */
|
|
|
|
mbox->controller.txdone_irq = false;
|
2017-08-01 18:35:54 +08:00
|
|
|
mbox->controller.txdone_poll = false;
|
2017-03-15 14:40:00 +08:00
|
|
|
mbox->controller.ops = &flexrm_mbox_chan_ops;
|
|
|
|
mbox->controller.dev = dev;
|
|
|
|
mbox->controller.num_chans = mbox->num_rings;
|
|
|
|
mbox->controller.of_xlate = flexrm_mbox_of_xlate;
|
|
|
|
mbox->controller.chans = devm_kcalloc(dev, mbox->num_rings,
|
|
|
|
sizeof(*mbox->controller.chans), GFP_KERNEL);
|
|
|
|
if (!mbox->controller.chans) {
|
|
|
|
ret = -ENOMEM;
|
2017-08-01 18:35:51 +08:00
|
|
|
goto fail_free_debugfs_root;
|
2017-03-15 14:40:00 +08:00
|
|
|
}
|
|
|
|
for (index = 0; index < mbox->num_rings; index++)
|
|
|
|
mbox->controller.chans[index].con_priv = &mbox->rings[index];
|
|
|
|
|
|
|
|
/* Register mailbox controller */
|
|
|
|
ret = mbox_controller_register(&mbox->controller);
|
|
|
|
if (ret)
|
2017-08-01 18:35:51 +08:00
|
|
|
goto fail_free_debugfs_root;
|
2017-03-15 14:40:00 +08:00
|
|
|
|
|
|
|
dev_info(dev, "registered flexrm mailbox with %d channels\n",
|
|
|
|
mbox->controller.num_chans);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
2017-08-01 18:35:51 +08:00
|
|
|
fail_free_debugfs_root:
|
|
|
|
debugfs_remove_recursive(mbox->root);
|
2017-03-15 14:40:00 +08:00
|
|
|
fail_free_msis:
|
|
|
|
platform_msi_domain_free_irqs(dev);
|
|
|
|
fail_destroy_cmpl_pool:
|
|
|
|
dma_pool_destroy(mbox->cmpl_pool);
|
|
|
|
fail_destroy_bd_pool:
|
|
|
|
dma_pool_destroy(mbox->bd_pool);
|
|
|
|
fail:
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int flexrm_mbox_remove(struct platform_device *pdev)
|
|
|
|
{
|
|
|
|
struct device *dev = &pdev->dev;
|
|
|
|
struct flexrm_mbox *mbox = platform_get_drvdata(pdev);
|
|
|
|
|
|
|
|
mbox_controller_unregister(&mbox->controller);
|
|
|
|
|
2017-08-01 18:35:51 +08:00
|
|
|
debugfs_remove_recursive(mbox->root);
|
|
|
|
|
2017-03-15 14:40:00 +08:00
|
|
|
platform_msi_domain_free_irqs(dev);
|
|
|
|
|
|
|
|
dma_pool_destroy(mbox->cmpl_pool);
|
|
|
|
dma_pool_destroy(mbox->bd_pool);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static const struct of_device_id flexrm_mbox_of_match[] = {
|
|
|
|
{ .compatible = "brcm,iproc-flexrm-mbox", },
|
|
|
|
{},
|
|
|
|
};
|
|
|
|
MODULE_DEVICE_TABLE(of, flexrm_mbox_of_match);
|
|
|
|
|
|
|
|
static struct platform_driver flexrm_mbox_driver = {
|
|
|
|
.driver = {
|
|
|
|
.name = "brcm-flexrm-mbox",
|
|
|
|
.of_match_table = flexrm_mbox_of_match,
|
|
|
|
},
|
|
|
|
.probe = flexrm_mbox_probe,
|
|
|
|
.remove = flexrm_mbox_remove,
|
|
|
|
};
|
|
|
|
module_platform_driver(flexrm_mbox_driver);
|
|
|
|
|
|
|
|
MODULE_AUTHOR("Anup Patel <anup.patel@broadcom.com>");
|
|
|
|
MODULE_DESCRIPTION("Broadcom FlexRM mailbox driver");
|
|
|
|
MODULE_LICENSE("GPL v2");
|