478 lines
16 KiB
C
478 lines
16 KiB
C
/*
|
|
* Intel MIC Platform Software Stack (MPSS)
|
|
*
|
|
* This file is provided under a dual BSD/GPLv2 license. When using or
|
|
* redistributing this file, you may do so under either license.
|
|
*
|
|
* GPL LICENSE SUMMARY
|
|
*
|
|
* Copyright(c) 2015 Intel Corporation.
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of version 2 of the GNU General Public License as
|
|
* published by the Free Software Foundation.
|
|
*
|
|
* This program is distributed in the hope that it will be useful, but
|
|
* WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
* General Public License for more details.
|
|
*
|
|
* BSD LICENSE
|
|
*
|
|
* Copyright(c) 2015 Intel Corporation.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
*
|
|
* * Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* * Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in
|
|
* the documentation and/or other materials provided with the
|
|
* distribution.
|
|
* * Neither the name of Intel Corporation nor the names of its
|
|
* contributors may be used to endorse or promote products derived
|
|
* from this software without specific prior written permission.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
*
|
|
* Intel SCIF driver.
|
|
*
|
|
*/
|
|
#ifndef SCIF_RMA_H
|
|
#define SCIF_RMA_H
|
|
|
|
#include <linux/intel-iommu.h>
|
|
#include <linux/mmu_notifier.h>
|
|
|
|
#include "../bus/scif_bus.h"
|
|
|
|
/* If this bit is set then the mark is a remote fence mark */
|
|
#define SCIF_REMOTE_FENCE_BIT 31
|
|
/* Magic value used to indicate a remote fence request */
|
|
#define SCIF_REMOTE_FENCE BIT_ULL(SCIF_REMOTE_FENCE_BIT)
|
|
|
|
#define SCIF_MAX_UNALIGNED_BUF_SIZE (1024 * 1024ULL)
|
|
#define SCIF_KMEM_UNALIGNED_BUF_SIZE (SCIF_MAX_UNALIGNED_BUF_SIZE + \
|
|
(L1_CACHE_BYTES << 1))
|
|
|
|
#define SCIF_IOVA_START_PFN (1)
|
|
#define SCIF_IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
|
|
#define SCIF_DMA_64BIT_PFN SCIF_IOVA_PFN(DMA_BIT_MASK(64))
|
|
#define SCIF_DMA_63BIT_PFN SCIF_IOVA_PFN(DMA_BIT_MASK(63))
|
|
|
|
/*
|
|
* struct scif_endpt_rma_info - Per Endpoint Remote Memory Access Information
|
|
*
|
|
* @reg_list: List of registration windows for self
|
|
* @remote_reg_list: List of registration windows for peer
|
|
* @iovad: Offset generator
|
|
* @rma_lock: Synchronizes access to self/remote list and also protects the
|
|
* window from being destroyed while RMAs are in progress.
|
|
* @tc_lock: Synchronizes access to temporary cached windows list
|
|
* for SCIF Registration Caching.
|
|
* @mmn_lock: Synchronizes access to the list of MMU notifiers registered
|
|
* @tw_refcount: Keeps track of number of outstanding temporary registered
|
|
* windows created by scif_vreadfrom/scif_vwriteto which have
|
|
* not been destroyed.
|
|
* @tcw_refcount: Same as tw_refcount but for temporary cached windows
|
|
* @tcw_total_pages: Same as tcw_refcount but in terms of pages pinned
|
|
* @mmn_list: MMU notifier so that we can destroy the windows when required
|
|
* @fence_refcount: Keeps track of number of outstanding remote fence
|
|
* requests which have been received by the peer.
|
|
* @dma_chan: DMA channel used for all DMA transfers for this endpoint.
|
|
* @async_list_del: Detect asynchronous list entry deletion
|
|
* @vma_list: List of vmas with remote memory mappings
|
|
* @markwq: Wait queue used for scif_fence_mark/scif_fence_wait
|
|
*/
|
|
struct scif_endpt_rma_info {
|
|
struct list_head reg_list;
|
|
struct list_head remote_reg_list;
|
|
struct iova_domain iovad;
|
|
struct mutex rma_lock;
|
|
spinlock_t tc_lock;
|
|
struct mutex mmn_lock;
|
|
atomic_t tw_refcount;
|
|
atomic_t tcw_refcount;
|
|
atomic_t tcw_total_pages;
|
|
struct list_head mmn_list;
|
|
atomic_t fence_refcount;
|
|
struct dma_chan *dma_chan;
|
|
int async_list_del;
|
|
struct list_head vma_list;
|
|
wait_queue_head_t markwq;
|
|
};
|
|
|
|
/*
|
|
* struct scif_fence_info - used for tracking fence requests
|
|
*
|
|
* @state: State of this transfer
|
|
* @wq: Fences wait on this queue
|
|
* @dma_mark: Used for storing the DMA mark
|
|
*/
|
|
struct scif_fence_info {
|
|
enum scif_msg_state state;
|
|
struct completion comp;
|
|
int dma_mark;
|
|
};
|
|
|
|
/*
|
|
* struct scif_remote_fence_info - used for tracking remote fence requests
|
|
*
|
|
* @msg: List of SCIF node QP fence messages
|
|
* @list: Link to list of remote fence requests
|
|
*/
|
|
struct scif_remote_fence_info {
|
|
struct scifmsg msg;
|
|
struct list_head list;
|
|
};
|
|
|
|
/*
|
|
* Specifies whether an RMA operation can span across partial windows, a single
|
|
* window or multiple contiguous windows. Mmaps can span across partial windows.
|
|
* Unregistration can span across complete windows. scif_get_pages() can span a
|
|
* single window. A window can also be of type self or peer.
|
|
*/
|
|
enum scif_window_type {
|
|
SCIF_WINDOW_PARTIAL,
|
|
SCIF_WINDOW_SINGLE,
|
|
SCIF_WINDOW_FULL,
|
|
SCIF_WINDOW_SELF,
|
|
SCIF_WINDOW_PEER
|
|
};
|
|
|
|
/* The number of physical addresses that can be stored in a PAGE. */
|
|
#define SCIF_NR_ADDR_IN_PAGE (0x1000 >> 3)
|
|
|
|
/*
|
|
* struct scif_rma_lookup - RMA lookup data structure for page list transfers
|
|
*
|
|
* Store an array of lookup offsets. Each offset in this array maps
|
|
* one 4K page containing 512 physical addresses i.e. 2MB. 512 such
|
|
* offsets in a 4K page will correspond to 1GB of registered address space.
|
|
|
|
* @lookup: Array of offsets
|
|
* @offset: DMA offset of lookup array
|
|
*/
|
|
struct scif_rma_lookup {
|
|
dma_addr_t *lookup;
|
|
dma_addr_t offset;
|
|
};
|
|
|
|
/*
|
|
* struct scif_pinned_pages - A set of pinned pages obtained with
|
|
* scif_pin_pages() which could be part of multiple registered
|
|
* windows across different end points.
|
|
*
|
|
* @nr_pages: Number of pages which is defined as a s64 instead of an int
|
|
* to avoid sign extension with buffers >= 2GB
|
|
* @prot: read/write protections
|
|
* @map_flags: Flags specified during the pin operation
|
|
* @ref_count: Reference count bumped in terms of number of pages
|
|
* @magic: A magic value
|
|
* @pages: Array of pointers to struct pages populated with get_user_pages(..)
|
|
*/
|
|
struct scif_pinned_pages {
|
|
s64 nr_pages;
|
|
int prot;
|
|
int map_flags;
|
|
atomic_t ref_count;
|
|
u64 magic;
|
|
struct page **pages;
|
|
};
|
|
|
|
/*
|
|
* struct scif_status - Stores DMA status update information
|
|
*
|
|
* @src_dma_addr: Source buffer DMA address
|
|
* @val: src location for value to be written to the destination
|
|
* @ep: SCIF endpoint
|
|
*/
|
|
struct scif_status {
|
|
dma_addr_t src_dma_addr;
|
|
u64 val;
|
|
struct scif_endpt *ep;
|
|
};
|
|
|
|
/*
|
|
* struct scif_cb_arg - Stores the argument of the callback func
|
|
*
|
|
* @src_dma_addr: Source buffer DMA address
|
|
* @status: DMA status
|
|
* @ep: SCIF endpoint
|
|
*/
|
|
struct scif_cb_arg {
|
|
dma_addr_t src_dma_addr;
|
|
struct scif_status *status;
|
|
struct scif_endpt *ep;
|
|
};
|
|
|
|
/*
|
|
* struct scif_window - Registration Window for Self and Remote
|
|
*
|
|
* @nr_pages: Number of pages which is defined as a s64 instead of an int
|
|
* to avoid sign extension with buffers >= 2GB
|
|
* @nr_contig_chunks: Number of contiguous physical chunks
|
|
* @prot: read/write protections
|
|
* @ref_count: reference count in terms of number of pages
|
|
* @magic: Cookie to detect corruption
|
|
* @offset: registered offset
|
|
* @va_for_temp: va address that this window represents
|
|
* @dma_mark: Used to determine if all DMAs against the window are done
|
|
* @ep: Pointer to EP. Useful for passing EP around with messages to
|
|
avoid expensive list traversals.
|
|
* @list: link to list of windows for the endpoint
|
|
* @type: self or peer window
|
|
* @peer_window: Pointer to peer window. Useful for sending messages to peer
|
|
* without requiring an extra list traversal
|
|
* @unreg_state: unregistration state
|
|
* @offset_freed: True if the offset has been freed
|
|
* @temp: True for temporary windows created via scif_vreadfrom/scif_vwriteto
|
|
* @mm: memory descriptor for the task_struct which initiated the RMA
|
|
* @st: scatter gather table for DMA mappings with IOMMU enabled
|
|
* @pinned_pages: The set of pinned_pages backing this window
|
|
* @alloc_handle: Handle for sending ALLOC_REQ
|
|
* @regwq: Wait Queue for an registration (N)ACK
|
|
* @reg_state: Registration state
|
|
* @unregwq: Wait Queue for an unregistration (N)ACK
|
|
* @dma_addr_lookup: Lookup for physical addresses used for DMA
|
|
* @nr_lookup: Number of entries in lookup
|
|
* @mapped_offset: Offset used to map the window by the peer
|
|
* @dma_addr: Array of physical addresses used for Mgmt node & MIC initiated DMA
|
|
* @num_pages: Array specifying number of pages for each physical address
|
|
*/
|
|
struct scif_window {
|
|
s64 nr_pages;
|
|
int nr_contig_chunks;
|
|
int prot;
|
|
int ref_count;
|
|
u64 magic;
|
|
s64 offset;
|
|
unsigned long va_for_temp;
|
|
int dma_mark;
|
|
u64 ep;
|
|
struct list_head list;
|
|
enum scif_window_type type;
|
|
u64 peer_window;
|
|
enum scif_msg_state unreg_state;
|
|
bool offset_freed;
|
|
bool temp;
|
|
struct mm_struct *mm;
|
|
struct sg_table *st;
|
|
union {
|
|
struct {
|
|
struct scif_pinned_pages *pinned_pages;
|
|
struct scif_allocmsg alloc_handle;
|
|
wait_queue_head_t regwq;
|
|
enum scif_msg_state reg_state;
|
|
wait_queue_head_t unregwq;
|
|
};
|
|
struct {
|
|
struct scif_rma_lookup dma_addr_lookup;
|
|
struct scif_rma_lookup num_pages_lookup;
|
|
int nr_lookup;
|
|
dma_addr_t mapped_offset;
|
|
};
|
|
};
|
|
dma_addr_t *dma_addr;
|
|
u64 *num_pages;
|
|
} __packed;
|
|
|
|
/*
|
|
* scif_mmu_notif - SCIF mmu notifier information
|
|
*
|
|
* @mmu_notifier ep_mmu_notifier: MMU notifier operations
|
|
* @tc_reg_list: List of temp registration windows for self
|
|
* @mm: memory descriptor for the task_struct which initiated the RMA
|
|
* @ep: SCIF endpoint
|
|
* @list: link to list of MMU notifier information
|
|
*/
|
|
struct scif_mmu_notif {
|
|
#ifdef CONFIG_MMU_NOTIFIER
|
|
struct mmu_notifier ep_mmu_notifier;
|
|
#endif
|
|
struct list_head tc_reg_list;
|
|
struct mm_struct *mm;
|
|
struct scif_endpt *ep;
|
|
struct list_head list;
|
|
};
|
|
|
|
enum scif_rma_dir {
|
|
SCIF_LOCAL_TO_REMOTE,
|
|
SCIF_REMOTE_TO_LOCAL
|
|
};
|
|
|
|
extern struct kmem_cache *unaligned_cache;
|
|
/* Initialize RMA for this EP */
|
|
void scif_rma_ep_init(struct scif_endpt *ep);
|
|
/* Check if epd can be uninitialized */
|
|
int scif_rma_ep_can_uninit(struct scif_endpt *ep);
|
|
/* Obtain a new offset. Callee must grab RMA lock */
|
|
int scif_get_window_offset(struct scif_endpt *ep, int flags,
|
|
s64 offset, int nr_pages, s64 *out_offset);
|
|
/* Free offset. Callee must grab RMA lock */
|
|
void scif_free_window_offset(struct scif_endpt *ep,
|
|
struct scif_window *window, s64 offset);
|
|
/* Create self registration window */
|
|
struct scif_window *scif_create_window(struct scif_endpt *ep, int nr_pages,
|
|
s64 offset, bool temp);
|
|
/* Destroy self registration window.*/
|
|
int scif_destroy_window(struct scif_endpt *ep, struct scif_window *window);
|
|
void scif_unmap_window(struct scif_dev *remote_dev, struct scif_window *window);
|
|
/* Map pages of self window to Aperture/PCI */
|
|
int scif_map_window(struct scif_dev *remote_dev,
|
|
struct scif_window *window);
|
|
/* Unregister a self window */
|
|
int scif_unregister_window(struct scif_window *window);
|
|
/* Destroy remote registration window */
|
|
void
|
|
scif_destroy_remote_window(struct scif_window *window);
|
|
/* remove valid remote memory mappings from process address space */
|
|
void scif_zap_mmaps(int node);
|
|
/* Query if any applications have remote memory mappings */
|
|
bool scif_rma_do_apps_have_mmaps(int node);
|
|
/* Cleanup remote registration lists for zombie endpoints */
|
|
void scif_cleanup_rma_for_zombies(int node);
|
|
/* Reserve a DMA channel for a particular endpoint */
|
|
int scif_reserve_dma_chan(struct scif_endpt *ep);
|
|
/* Setup a DMA mark for an endpoint */
|
|
int _scif_fence_mark(scif_epd_t epd, int *mark);
|
|
int scif_prog_signal(scif_epd_t epd, off_t offset, u64 val,
|
|
enum scif_window_type type);
|
|
void scif_alloc_req(struct scif_dev *scifdev, struct scifmsg *msg);
|
|
void scif_alloc_gnt_rej(struct scif_dev *scifdev, struct scifmsg *msg);
|
|
void scif_free_virt(struct scif_dev *scifdev, struct scifmsg *msg);
|
|
void scif_recv_reg(struct scif_dev *scifdev, struct scifmsg *msg);
|
|
void scif_recv_unreg(struct scif_dev *scifdev, struct scifmsg *msg);
|
|
void scif_recv_reg_ack(struct scif_dev *scifdev, struct scifmsg *msg);
|
|
void scif_recv_reg_nack(struct scif_dev *scifdev, struct scifmsg *msg);
|
|
void scif_recv_unreg_ack(struct scif_dev *scifdev, struct scifmsg *msg);
|
|
void scif_recv_unreg_nack(struct scif_dev *scifdev, struct scifmsg *msg);
|
|
void scif_recv_munmap(struct scif_dev *scifdev, struct scifmsg *msg);
|
|
void scif_recv_mark(struct scif_dev *scifdev, struct scifmsg *msg);
|
|
void scif_recv_mark_resp(struct scif_dev *scifdev, struct scifmsg *msg);
|
|
void scif_recv_wait(struct scif_dev *scifdev, struct scifmsg *msg);
|
|
void scif_recv_wait_resp(struct scif_dev *scifdev, struct scifmsg *msg);
|
|
void scif_recv_sig_local(struct scif_dev *scifdev, struct scifmsg *msg);
|
|
void scif_recv_sig_remote(struct scif_dev *scifdev, struct scifmsg *msg);
|
|
void scif_recv_sig_resp(struct scif_dev *scifdev, struct scifmsg *msg);
|
|
void scif_mmu_notif_handler(struct work_struct *work);
|
|
void scif_rma_handle_remote_fences(void);
|
|
void scif_rma_destroy_windows(void);
|
|
void scif_rma_destroy_tcw_invalid(void);
|
|
int scif_drain_dma_intr(struct scif_hw_dev *sdev, struct dma_chan *chan);
|
|
|
|
struct scif_window_iter {
|
|
s64 offset;
|
|
int index;
|
|
};
|
|
|
|
static inline void
|
|
scif_init_window_iter(struct scif_window *window, struct scif_window_iter *iter)
|
|
{
|
|
iter->offset = window->offset;
|
|
iter->index = 0;
|
|
}
|
|
|
|
dma_addr_t scif_off_to_dma_addr(struct scif_window *window, s64 off,
|
|
size_t *nr_bytes,
|
|
struct scif_window_iter *iter);
|
|
static inline
|
|
dma_addr_t __scif_off_to_dma_addr(struct scif_window *window, s64 off)
|
|
{
|
|
return scif_off_to_dma_addr(window, off, NULL, NULL);
|
|
}
|
|
|
|
static inline bool scif_unaligned(off_t src_offset, off_t dst_offset)
|
|
{
|
|
src_offset = src_offset & (L1_CACHE_BYTES - 1);
|
|
dst_offset = dst_offset & (L1_CACHE_BYTES - 1);
|
|
return !(src_offset == dst_offset);
|
|
}
|
|
|
|
/*
|
|
* scif_zalloc:
|
|
* @size: Size of the allocation request.
|
|
*
|
|
* Helper API which attempts to allocate zeroed pages via
|
|
* __get_free_pages(..) first and then falls back on
|
|
* vzalloc(..) if that fails.
|
|
*/
|
|
static inline void *scif_zalloc(size_t size)
|
|
{
|
|
void *ret = NULL;
|
|
size_t align = ALIGN(size, PAGE_SIZE);
|
|
|
|
if (align && get_order(align) < MAX_ORDER)
|
|
ret = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
|
|
get_order(align));
|
|
return ret ? ret : vzalloc(align);
|
|
}
|
|
|
|
/*
|
|
* scif_free:
|
|
* @addr: Address to be freed.
|
|
* @size: Size of the allocation.
|
|
* Helper API which frees memory allocated via scif_zalloc().
|
|
*/
|
|
static inline void scif_free(void *addr, size_t size)
|
|
{
|
|
size_t align = ALIGN(size, PAGE_SIZE);
|
|
|
|
if (is_vmalloc_addr(addr))
|
|
vfree(addr);
|
|
else
|
|
free_pages((unsigned long)addr, get_order(align));
|
|
}
|
|
|
|
static inline void scif_get_window(struct scif_window *window, int nr_pages)
|
|
{
|
|
window->ref_count += nr_pages;
|
|
}
|
|
|
|
static inline void scif_put_window(struct scif_window *window, int nr_pages)
|
|
{
|
|
window->ref_count -= nr_pages;
|
|
}
|
|
|
|
static inline void scif_set_window_ref(struct scif_window *window, int nr_pages)
|
|
{
|
|
window->ref_count = nr_pages;
|
|
}
|
|
|
|
static inline void
|
|
scif_queue_for_cleanup(struct scif_window *window, struct list_head *list)
|
|
{
|
|
spin_lock(&scif_info.rmalock);
|
|
list_add_tail(&window->list, list);
|
|
spin_unlock(&scif_info.rmalock);
|
|
schedule_work(&scif_info.misc_work);
|
|
}
|
|
|
|
static inline void __scif_rma_destroy_tcw_helper(struct scif_window *window)
|
|
{
|
|
list_del_init(&window->list);
|
|
scif_queue_for_cleanup(window, &scif_info.rma_tc);
|
|
}
|
|
|
|
static inline bool scif_is_iommu_enabled(void)
|
|
{
|
|
#ifdef CONFIG_INTEL_IOMMU
|
|
return intel_iommu_enabled;
|
|
#else
|
|
return false;
|
|
#endif
|
|
}
|
|
#endif /* SCIF_RMA_H */
|