From 6296883ca4cd52dafb45f191d24102e28ded38f2 Mon Sep 17 00:00:00 2001 From: Yevgeny Petrilin Date: Wed, 23 Apr 2008 11:55:45 -0700 Subject: [PATCH] mlx4_core: Move kernel doorbell management into core In addition to mlx4_ib, there will be ethernet and FC consumers of mlx4_core, so move the code for managing kernel doorbells into the core module to avoid having to duplicate this multiple times. Signed-off-by: Yevgeny Petrilin Signed-off-by: Roland Dreier --- drivers/infiniband/hw/mlx4/cq.c | 6 +- drivers/infiniband/hw/mlx4/doorbell.c | 122 +------------------------- drivers/infiniband/hw/mlx4/main.c | 3 - drivers/infiniband/hw/mlx4/mlx4_ib.h | 33 ++----- drivers/infiniband/hw/mlx4/qp.c | 6 +- drivers/infiniband/hw/mlx4/srq.c | 6 +- drivers/net/mlx4/alloc.c | 111 +++++++++++++++++++++++ drivers/net/mlx4/main.c | 3 + drivers/net/mlx4/mlx4.h | 3 + include/linux/mlx4/device.h | 29 ++++++ 10 files changed, 162 insertions(+), 160 deletions(-) diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c index 3557e7edc9b6..5e570bb0bb6f 100644 --- a/drivers/infiniband/hw/mlx4/cq.c +++ b/drivers/infiniband/hw/mlx4/cq.c @@ -204,7 +204,7 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector uar = &to_mucontext(context)->uar; } else { - err = mlx4_ib_db_alloc(dev, &cq->db, 1); + err = mlx4_db_alloc(dev->dev, &cq->db, 1); if (err) goto err_cq; @@ -250,7 +250,7 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector err_db: if (!context) - mlx4_ib_db_free(dev, &cq->db); + mlx4_db_free(dev->dev, &cq->db); err_cq: kfree(cq); @@ -435,7 +435,7 @@ int mlx4_ib_destroy_cq(struct ib_cq *cq) ib_umem_release(mcq->umem); } else { mlx4_ib_free_cq_buf(dev, &mcq->buf, cq->cqe + 1); - mlx4_ib_db_free(dev, &mcq->db); + mlx4_db_free(dev->dev, &mcq->db); } kfree(mcq); diff --git a/drivers/infiniband/hw/mlx4/doorbell.c b/drivers/infiniband/hw/mlx4/doorbell.c index 1c36087aef14..8e342cc9baec 100644 --- a/drivers/infiniband/hw/mlx4/doorbell.c +++ b/drivers/infiniband/hw/mlx4/doorbell.c @@ -34,124 +34,6 @@ #include "mlx4_ib.h" -struct mlx4_ib_db_pgdir { - struct list_head list; - DECLARE_BITMAP(order0, MLX4_IB_DB_PER_PAGE); - DECLARE_BITMAP(order1, MLX4_IB_DB_PER_PAGE / 2); - unsigned long *bits[2]; - __be32 *db_page; - dma_addr_t db_dma; -}; - -static struct mlx4_ib_db_pgdir *mlx4_ib_alloc_db_pgdir(struct mlx4_ib_dev *dev) -{ - struct mlx4_ib_db_pgdir *pgdir; - - pgdir = kzalloc(sizeof *pgdir, GFP_KERNEL); - if (!pgdir) - return NULL; - - bitmap_fill(pgdir->order1, MLX4_IB_DB_PER_PAGE / 2); - pgdir->bits[0] = pgdir->order0; - pgdir->bits[1] = pgdir->order1; - pgdir->db_page = dma_alloc_coherent(dev->ib_dev.dma_device, - PAGE_SIZE, &pgdir->db_dma, - GFP_KERNEL); - if (!pgdir->db_page) { - kfree(pgdir); - return NULL; - } - - return pgdir; -} - -static int mlx4_ib_alloc_db_from_pgdir(struct mlx4_ib_db_pgdir *pgdir, - struct mlx4_ib_db *db, int order) -{ - int o; - int i; - - for (o = order; o <= 1; ++o) { - i = find_first_bit(pgdir->bits[o], MLX4_IB_DB_PER_PAGE >> o); - if (i < MLX4_IB_DB_PER_PAGE >> o) - goto found; - } - - return -ENOMEM; - -found: - clear_bit(i, pgdir->bits[o]); - - i <<= o; - - if (o > order) - set_bit(i ^ 1, pgdir->bits[order]); - - db->u.pgdir = pgdir; - db->index = i; - db->db = pgdir->db_page + db->index; - db->dma = pgdir->db_dma + db->index * 4; - db->order = order; - - return 0; -} - -int mlx4_ib_db_alloc(struct mlx4_ib_dev *dev, struct mlx4_ib_db *db, int order) -{ - struct mlx4_ib_db_pgdir *pgdir; - int ret = 0; - - mutex_lock(&dev->pgdir_mutex); - - list_for_each_entry(pgdir, &dev->pgdir_list, list) - if (!mlx4_ib_alloc_db_from_pgdir(pgdir, db, order)) - goto out; - - pgdir = mlx4_ib_alloc_db_pgdir(dev); - if (!pgdir) { - ret = -ENOMEM; - goto out; - } - - list_add(&pgdir->list, &dev->pgdir_list); - - /* This should never fail -- we just allocated an empty page: */ - WARN_ON(mlx4_ib_alloc_db_from_pgdir(pgdir, db, order)); - -out: - mutex_unlock(&dev->pgdir_mutex); - - return ret; -} - -void mlx4_ib_db_free(struct mlx4_ib_dev *dev, struct mlx4_ib_db *db) -{ - int o; - int i; - - mutex_lock(&dev->pgdir_mutex); - - o = db->order; - i = db->index; - - if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) { - clear_bit(i ^ 1, db->u.pgdir->order0); - ++o; - } - - i >>= o; - set_bit(i, db->u.pgdir->bits[o]); - - if (bitmap_full(db->u.pgdir->order1, MLX4_IB_DB_PER_PAGE / 2)) { - dma_free_coherent(dev->ib_dev.dma_device, PAGE_SIZE, - db->u.pgdir->db_page, db->u.pgdir->db_dma); - list_del(&db->u.pgdir->list); - kfree(db->u.pgdir); - } - - mutex_unlock(&dev->pgdir_mutex); -} - struct mlx4_ib_user_db_page { struct list_head list; struct ib_umem *umem; @@ -160,7 +42,7 @@ struct mlx4_ib_user_db_page { }; int mlx4_ib_db_map_user(struct mlx4_ib_ucontext *context, unsigned long virt, - struct mlx4_ib_db *db) + struct mlx4_db *db) { struct mlx4_ib_user_db_page *page; struct ib_umem_chunk *chunk; @@ -202,7 +84,7 @@ int mlx4_ib_db_map_user(struct mlx4_ib_ucontext *context, unsigned long virt, return err; } -void mlx4_ib_db_unmap_user(struct mlx4_ib_ucontext *context, struct mlx4_ib_db *db) +void mlx4_ib_db_unmap_user(struct mlx4_ib_ucontext *context, struct mlx4_db *db) { mutex_lock(&context->db_page_mutex); diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c index 4d9b5ac42202..4d61e32866c6 100644 --- a/drivers/infiniband/hw/mlx4/main.c +++ b/drivers/infiniband/hw/mlx4/main.c @@ -557,9 +557,6 @@ static void *mlx4_ib_add(struct mlx4_dev *dev) goto err_uar; MLX4_INIT_DOORBELL_LOCK(&ibdev->uar_lock); - INIT_LIST_HEAD(&ibdev->pgdir_list); - mutex_init(&ibdev->pgdir_mutex); - ibdev->dev = dev; strlcpy(ibdev->ib_dev.name, "mlx4_%d", IB_DEVICE_NAME_MAX); diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h index 9e637323c155..5cf994794d25 100644 --- a/drivers/infiniband/hw/mlx4/mlx4_ib.h +++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h @@ -43,24 +43,6 @@ #include #include -enum { - MLX4_IB_DB_PER_PAGE = PAGE_SIZE / 4 -}; - -struct mlx4_ib_db_pgdir; -struct mlx4_ib_user_db_page; - -struct mlx4_ib_db { - __be32 *db; - union { - struct mlx4_ib_db_pgdir *pgdir; - struct mlx4_ib_user_db_page *user_page; - } u; - dma_addr_t dma; - int index; - int order; -}; - struct mlx4_ib_ucontext { struct ib_ucontext ibucontext; struct mlx4_uar uar; @@ -88,7 +70,7 @@ struct mlx4_ib_cq { struct mlx4_cq mcq; struct mlx4_ib_cq_buf buf; struct mlx4_ib_cq_resize *resize_buf; - struct mlx4_ib_db db; + struct mlx4_db db; spinlock_t lock; struct mutex resize_mutex; struct ib_umem *umem; @@ -127,7 +109,7 @@ struct mlx4_ib_qp { struct mlx4_qp mqp; struct mlx4_buf buf; - struct mlx4_ib_db db; + struct mlx4_db db; struct mlx4_ib_wq rq; u32 doorbell_qpn; @@ -154,7 +136,7 @@ struct mlx4_ib_srq { struct ib_srq ibsrq; struct mlx4_srq msrq; struct mlx4_buf buf; - struct mlx4_ib_db db; + struct mlx4_db db; u64 *wrid; spinlock_t lock; int head; @@ -175,9 +157,6 @@ struct mlx4_ib_dev { struct mlx4_dev *dev; void __iomem *uar_map; - struct list_head pgdir_list; - struct mutex pgdir_mutex; - struct mlx4_uar priv_uar; u32 priv_pdn; MLX4_DECLARE_DOORBELL_LOCK(uar_lock); @@ -248,11 +227,9 @@ static inline struct mlx4_ib_ah *to_mah(struct ib_ah *ibah) return container_of(ibah, struct mlx4_ib_ah, ibah); } -int mlx4_ib_db_alloc(struct mlx4_ib_dev *dev, struct mlx4_ib_db *db, int order); -void mlx4_ib_db_free(struct mlx4_ib_dev *dev, struct mlx4_ib_db *db); int mlx4_ib_db_map_user(struct mlx4_ib_ucontext *context, unsigned long virt, - struct mlx4_ib_db *db); -void mlx4_ib_db_unmap_user(struct mlx4_ib_ucontext *context, struct mlx4_ib_db *db); + struct mlx4_db *db); +void mlx4_ib_db_unmap_user(struct mlx4_ib_ucontext *context, struct mlx4_db *db); struct ib_mr *mlx4_ib_get_dma_mr(struct ib_pd *pd, int acc); int mlx4_ib_umem_write_mtt(struct mlx4_ib_dev *dev, struct mlx4_mtt *mtt, diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c index b75efae7e449..80ea8b9e7761 100644 --- a/drivers/infiniband/hw/mlx4/qp.c +++ b/drivers/infiniband/hw/mlx4/qp.c @@ -514,7 +514,7 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd, goto err; if (!init_attr->srq) { - err = mlx4_ib_db_alloc(dev, &qp->db, 0); + err = mlx4_db_alloc(dev->dev, &qp->db, 0); if (err) goto err; @@ -580,7 +580,7 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd, err_db: if (!pd->uobject && !init_attr->srq) - mlx4_ib_db_free(dev, &qp->db); + mlx4_db_free(dev->dev, &qp->db); err: return err; @@ -666,7 +666,7 @@ static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp, kfree(qp->rq.wrid); mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf); if (!qp->ibqp.srq) - mlx4_ib_db_free(dev, &qp->db); + mlx4_db_free(dev->dev, &qp->db); } } diff --git a/drivers/infiniband/hw/mlx4/srq.c b/drivers/infiniband/hw/mlx4/srq.c index beaa3b06cf58..204619702f9d 100644 --- a/drivers/infiniband/hw/mlx4/srq.c +++ b/drivers/infiniband/hw/mlx4/srq.c @@ -129,7 +129,7 @@ struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd, if (err) goto err_mtt; } else { - err = mlx4_ib_db_alloc(dev, &srq->db, 0); + err = mlx4_db_alloc(dev->dev, &srq->db, 0); if (err) goto err_srq; @@ -200,7 +200,7 @@ struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd, err_db: if (!pd->uobject) - mlx4_ib_db_free(dev, &srq->db); + mlx4_db_free(dev->dev, &srq->db); err_srq: kfree(srq); @@ -267,7 +267,7 @@ int mlx4_ib_destroy_srq(struct ib_srq *srq) kfree(msrq->wrid); mlx4_buf_free(dev->dev, msrq->msrq.max << msrq->msrq.wqe_shift, &msrq->buf); - mlx4_ib_db_free(dev, &msrq->db); + mlx4_db_free(dev->dev, &msrq->db); } kfree(msrq); diff --git a/drivers/net/mlx4/alloc.c b/drivers/net/mlx4/alloc.c index 75ef9d0d974d..43c6d04bb880 100644 --- a/drivers/net/mlx4/alloc.c +++ b/drivers/net/mlx4/alloc.c @@ -196,3 +196,114 @@ void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf) } } EXPORT_SYMBOL_GPL(mlx4_buf_free); + +static struct mlx4_db_pgdir *mlx4_alloc_db_pgdir(struct device *dma_device) +{ + struct mlx4_db_pgdir *pgdir; + + pgdir = kzalloc(sizeof *pgdir, GFP_KERNEL); + if (!pgdir) + return NULL; + + bitmap_fill(pgdir->order1, MLX4_DB_PER_PAGE / 2); + pgdir->bits[0] = pgdir->order0; + pgdir->bits[1] = pgdir->order1; + pgdir->db_page = dma_alloc_coherent(dma_device, PAGE_SIZE, + &pgdir->db_dma, GFP_KERNEL); + if (!pgdir->db_page) { + kfree(pgdir); + return NULL; + } + + return pgdir; +} + +static int mlx4_alloc_db_from_pgdir(struct mlx4_db_pgdir *pgdir, + struct mlx4_db *db, int order) +{ + int o; + int i; + + for (o = order; o <= 1; ++o) { + i = find_first_bit(pgdir->bits[o], MLX4_DB_PER_PAGE >> o); + if (i < MLX4_DB_PER_PAGE >> o) + goto found; + } + + return -ENOMEM; + +found: + clear_bit(i, pgdir->bits[o]); + + i <<= o; + + if (o > order) + set_bit(i ^ 1, pgdir->bits[order]); + + db->u.pgdir = pgdir; + db->index = i; + db->db = pgdir->db_page + db->index; + db->dma = pgdir->db_dma + db->index * 4; + db->order = order; + + return 0; +} + +int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_db_pgdir *pgdir; + int ret = 0; + + mutex_lock(&priv->pgdir_mutex); + + list_for_each_entry(pgdir, &priv->pgdir_list, list) + if (!mlx4_alloc_db_from_pgdir(pgdir, db, order)) + goto out; + + pgdir = mlx4_alloc_db_pgdir(&(dev->pdev->dev)); + if (!pgdir) { + ret = -ENOMEM; + goto out; + } + + list_add(&pgdir->list, &priv->pgdir_list); + + /* This should never fail -- we just allocated an empty page: */ + WARN_ON(mlx4_alloc_db_from_pgdir(pgdir, db, order)); + +out: + mutex_unlock(&priv->pgdir_mutex); + + return ret; +} +EXPORT_SYMBOL_GPL(mlx4_db_alloc); + +void mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int o; + int i; + + mutex_lock(&priv->pgdir_mutex); + + o = db->order; + i = db->index; + + if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) { + clear_bit(i ^ 1, db->u.pgdir->order0); + ++o; + } + i >>= o; + set_bit(i, db->u.pgdir->bits[o]); + + if (bitmap_full(db->u.pgdir->order1, MLX4_DB_PER_PAGE / 2)) { + dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE, + db->u.pgdir->db_page, db->u.pgdir->db_dma); + list_del(&db->u.pgdir->list); + kfree(db->u.pgdir); + } + + mutex_unlock(&priv->pgdir_mutex); +} +EXPORT_SYMBOL_GPL(mlx4_db_free); diff --git a/drivers/net/mlx4/main.c b/drivers/net/mlx4/main.c index 49a4acab5e82..a6aa49fc1d68 100644 --- a/drivers/net/mlx4/main.c +++ b/drivers/net/mlx4/main.c @@ -798,6 +798,9 @@ static int __mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id) INIT_LIST_HEAD(&priv->ctx_list); spin_lock_init(&priv->ctx_lock); + INIT_LIST_HEAD(&priv->pgdir_list); + mutex_init(&priv->pgdir_mutex); + /* * Now reset the HCA before we touch the PCI capabilities or * attempt a firmware command, since a boot ROM may have left diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h index 73336810e652..a4023c2dd050 100644 --- a/drivers/net/mlx4/mlx4.h +++ b/drivers/net/mlx4/mlx4.h @@ -257,6 +257,9 @@ struct mlx4_priv { struct list_head ctx_list; spinlock_t ctx_lock; + struct list_head pgdir_list; + struct mutex pgdir_mutex; + struct mlx4_fw fw; struct mlx4_cmd cmd; diff --git a/include/linux/mlx4/device.h b/include/linux/mlx4/device.h index ff7df1a2222f..0a47457931ac 100644 --- a/include/linux/mlx4/device.h +++ b/include/linux/mlx4/device.h @@ -208,6 +208,32 @@ struct mlx4_mtt { int page_shift; }; +enum { + MLX4_DB_PER_PAGE = PAGE_SIZE / 4 +}; + +struct mlx4_db_pgdir { + struct list_head list; + DECLARE_BITMAP(order0, MLX4_DB_PER_PAGE); + DECLARE_BITMAP(order1, MLX4_DB_PER_PAGE / 2); + unsigned long *bits[2]; + __be32 *db_page; + dma_addr_t db_dma; +}; + +struct mlx4_ib_user_db_page; + +struct mlx4_db { + __be32 *db; + union { + struct mlx4_db_pgdir *pgdir; + struct mlx4_ib_user_db_page *user_page; + } u; + dma_addr_t dma; + int index; + int order; +}; + struct mlx4_mr { struct mlx4_mtt mtt; u64 iova; @@ -341,6 +367,9 @@ int mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, int mlx4_buf_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, struct mlx4_buf *buf); +int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order); +void mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db); + int mlx4_cq_alloc(struct mlx4_dev *dev, int nent, struct mlx4_mtt *mtt, struct mlx4_uar *uar, u64 db_rec, struct mlx4_cq *cq); void mlx4_cq_free(struct mlx4_dev *dev, struct mlx4_cq *cq);