2018-05-18 20:00:21 +08:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0 */
|
|
|
|
/* XDP user-space ring structure
|
2018-05-02 19:01:24 +08:00
|
|
|
* Copyright(c) 2018 Intel Corporation.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifndef _LINUX_XSK_QUEUE_H
|
|
|
|
#define _LINUX_XSK_QUEUE_H
|
|
|
|
|
|
|
|
#include <linux/types.h>
|
|
|
|
#include <linux/if_xdp.h>
|
2018-06-04 20:05:51 +08:00
|
|
|
#include <net/xdp_sock.h>
|
2018-05-02 19:01:24 +08:00
|
|
|
|
2018-05-02 19:01:27 +08:00
|
|
|
#define RX_BATCH_SIZE 16
|
2018-06-04 20:05:57 +08:00
|
|
|
#define LAZY_UPDATE_THRESHOLD 128
|
2018-05-02 19:01:27 +08:00
|
|
|
|
xsk: remove explicit ring structure from uapi
In this commit we remove the explicit ring structure from the the
uapi. It is tricky for an uapi to depend on a certain L1 cache line
size, since it can differ for variants of the same architecture. Now,
we let the user application determine the offsets of the producer,
consumer and descriptors by asking the socket via getsockopt.
A typical flow would be (Rx ring):
struct xdp_mmap_offsets off;
struct xdp_desc *ring;
u32 *prod, *cons;
void *map;
...
getsockopt(fd, SOL_XDP, XDP_MMAP_OFFSETS, &off, &optlen);
map = mmap(NULL, off.rx.desc +
NUM_DESCS * sizeof(struct xdp_desc),
PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, sfd,
XDP_PGOFF_RX_RING);
prod = map + off.rx.producer;
cons = map + off.rx.consumer;
ring = map + off.rx.desc;
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-05-22 15:34:59 +08:00
|
|
|
struct xdp_ring {
|
|
|
|
u32 producer ____cacheline_aligned_in_smp;
|
|
|
|
u32 consumer ____cacheline_aligned_in_smp;
|
|
|
|
};
|
|
|
|
|
|
|
|
/* Used for the RX and TX queues for packets */
|
|
|
|
struct xdp_rxtx_ring {
|
|
|
|
struct xdp_ring ptrs;
|
|
|
|
struct xdp_desc desc[0] ____cacheline_aligned_in_smp;
|
|
|
|
};
|
|
|
|
|
|
|
|
/* Used for the fill and completion queues for buffers */
|
|
|
|
struct xdp_umem_ring {
|
|
|
|
struct xdp_ring ptrs;
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 19:57:13 +08:00
|
|
|
u64 desc[0] ____cacheline_aligned_in_smp;
|
xsk: remove explicit ring structure from uapi
In this commit we remove the explicit ring structure from the the
uapi. It is tricky for an uapi to depend on a certain L1 cache line
size, since it can differ for variants of the same architecture. Now,
we let the user application determine the offsets of the producer,
consumer and descriptors by asking the socket via getsockopt.
A typical flow would be (Rx ring):
struct xdp_mmap_offsets off;
struct xdp_desc *ring;
u32 *prod, *cons;
void *map;
...
getsockopt(fd, SOL_XDP, XDP_MMAP_OFFSETS, &off, &optlen);
map = mmap(NULL, off.rx.desc +
NUM_DESCS * sizeof(struct xdp_desc),
PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, sfd,
XDP_PGOFF_RX_RING);
prod = map + off.rx.producer;
cons = map + off.rx.consumer;
ring = map + off.rx.desc;
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-05-22 15:34:59 +08:00
|
|
|
};
|
|
|
|
|
2018-05-02 19:01:24 +08:00
|
|
|
struct xsk_queue {
|
2018-08-31 19:40:02 +08:00
|
|
|
u64 chunk_mask;
|
|
|
|
u64 size;
|
2018-05-02 19:01:24 +08:00
|
|
|
u32 ring_mask;
|
|
|
|
u32 nentries;
|
|
|
|
u32 prod_head;
|
|
|
|
u32 prod_tail;
|
|
|
|
u32 cons_head;
|
|
|
|
u32 cons_tail;
|
|
|
|
struct xdp_ring *ring;
|
|
|
|
u64 invalid_descs;
|
|
|
|
};
|
|
|
|
|
2019-04-16 20:58:08 +08:00
|
|
|
/* The structure of the shared state of the rings are the same as the
|
|
|
|
* ring buffer in kernel/events/ring_buffer.c. For the Rx and completion
|
|
|
|
* ring, the kernel is the producer and user space is the consumer. For
|
|
|
|
* the Tx and fill rings, the kernel is the consumer and user space is
|
|
|
|
* the producer.
|
|
|
|
*
|
|
|
|
* producer consumer
|
|
|
|
*
|
|
|
|
* if (LOAD ->consumer) { LOAD ->producer
|
|
|
|
* (A) smp_rmb() (C)
|
|
|
|
* STORE $data LOAD $data
|
|
|
|
* smp_wmb() (B) smp_mb() (D)
|
|
|
|
* STORE ->producer STORE ->consumer
|
|
|
|
* }
|
|
|
|
*
|
|
|
|
* (A) pairs with (D), and (B) pairs with (C).
|
|
|
|
*
|
|
|
|
* Starting with (B), it protects the data from being written after
|
|
|
|
* the producer pointer. If this barrier was missing, the consumer
|
|
|
|
* could observe the producer pointer being set and thus load the data
|
|
|
|
* before the producer has written the new data. The consumer would in
|
|
|
|
* this case load the old data.
|
|
|
|
*
|
|
|
|
* (C) protects the consumer from speculatively loading the data before
|
|
|
|
* the producer pointer actually has been read. If we do not have this
|
|
|
|
* barrier, some architectures could load old data as speculative loads
|
|
|
|
* are not discarded as the CPU does not know there is a dependency
|
|
|
|
* between ->producer and data.
|
|
|
|
*
|
|
|
|
* (A) is a control dependency that separates the load of ->consumer
|
|
|
|
* from the stores of $data. In case ->consumer indicates there is no
|
|
|
|
* room in the buffer to store $data we do not. So no barrier is needed.
|
|
|
|
*
|
|
|
|
* (D) protects the load of the data to be observed to happen after the
|
|
|
|
* store of the consumer pointer. If we did not have this memory
|
|
|
|
* barrier, the producer could observe the consumer pointer being set
|
|
|
|
* and overwrite the data with a new value before the consumer got the
|
|
|
|
* chance to read the old value. The consumer would thus miss reading
|
|
|
|
* the old entry and very likely read the new entry twice, once right
|
|
|
|
* now and again after circling through the ring.
|
|
|
|
*/
|
|
|
|
|
2018-05-02 19:01:27 +08:00
|
|
|
/* Common functions operating for both RXTX and umem queues */
|
|
|
|
|
2018-05-02 19:01:35 +08:00
|
|
|
static inline u64 xskq_nb_invalid_descs(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
return q ? q->invalid_descs : 0;
|
|
|
|
}
|
|
|
|
|
2018-05-02 19:01:27 +08:00
|
|
|
static inline u32 xskq_nb_avail(struct xsk_queue *q, u32 dcnt)
|
|
|
|
{
|
|
|
|
u32 entries = q->prod_tail - q->cons_tail;
|
|
|
|
|
|
|
|
if (entries == 0) {
|
|
|
|
/* Refresh the local pointer */
|
|
|
|
q->prod_tail = READ_ONCE(q->ring->producer);
|
|
|
|
entries = q->prod_tail - q->cons_tail;
|
|
|
|
}
|
|
|
|
|
|
|
|
return (entries > dcnt) ? dcnt : entries;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline u32 xskq_nb_free(struct xsk_queue *q, u32 producer, u32 dcnt)
|
|
|
|
{
|
2018-06-29 15:48:17 +08:00
|
|
|
u32 free_entries = q->nentries - (producer - q->cons_tail);
|
2018-05-02 19:01:27 +08:00
|
|
|
|
|
|
|
if (free_entries >= dcnt)
|
|
|
|
return free_entries;
|
|
|
|
|
|
|
|
/* Refresh the local tail pointer */
|
|
|
|
q->cons_tail = READ_ONCE(q->ring->consumer);
|
|
|
|
return q->nentries - (producer - q->cons_tail);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* UMEM queue */
|
|
|
|
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 19:57:13 +08:00
|
|
|
static inline bool xskq_is_valid_addr(struct xsk_queue *q, u64 addr)
|
2018-05-02 19:01:27 +08:00
|
|
|
{
|
2018-08-31 19:40:02 +08:00
|
|
|
if (addr >= q->size) {
|
2018-05-02 19:01:27 +08:00
|
|
|
q->invalid_descs++;
|
|
|
|
return false;
|
|
|
|
}
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 19:57:13 +08:00
|
|
|
|
2018-05-02 19:01:27 +08:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 19:57:13 +08:00
|
|
|
static inline u64 *xskq_validate_addr(struct xsk_queue *q, u64 *addr)
|
2018-05-02 19:01:27 +08:00
|
|
|
{
|
|
|
|
while (q->cons_tail != q->cons_head) {
|
|
|
|
struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;
|
|
|
|
unsigned int idx = q->cons_tail & q->ring_mask;
|
|
|
|
|
2018-08-31 19:40:02 +08:00
|
|
|
*addr = READ_ONCE(ring->desc[idx]) & q->chunk_mask;
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 19:57:13 +08:00
|
|
|
if (xskq_is_valid_addr(q, *addr))
|
|
|
|
return addr;
|
2018-05-02 19:01:27 +08:00
|
|
|
|
|
|
|
q->cons_tail++;
|
|
|
|
}
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 19:57:13 +08:00
|
|
|
static inline u64 *xskq_peek_addr(struct xsk_queue *q, u64 *addr)
|
2018-05-02 19:01:27 +08:00
|
|
|
{
|
|
|
|
if (q->cons_tail == q->cons_head) {
|
2019-04-16 20:58:08 +08:00
|
|
|
smp_mb(); /* D, matches A */
|
2018-05-02 19:01:27 +08:00
|
|
|
WRITE_ONCE(q->ring->consumer, q->cons_tail);
|
|
|
|
q->cons_head = q->cons_tail + xskq_nb_avail(q, RX_BATCH_SIZE);
|
|
|
|
|
|
|
|
/* Order consumer and data */
|
|
|
|
smp_rmb();
|
|
|
|
}
|
|
|
|
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 19:57:13 +08:00
|
|
|
return xskq_validate_addr(q, addr);
|
2018-05-02 19:01:27 +08:00
|
|
|
}
|
|
|
|
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 19:57:13 +08:00
|
|
|
static inline void xskq_discard_addr(struct xsk_queue *q)
|
2018-05-02 19:01:27 +08:00
|
|
|
{
|
|
|
|
q->cons_tail++;
|
|
|
|
}
|
|
|
|
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 19:57:13 +08:00
|
|
|
static inline int xskq_produce_addr(struct xsk_queue *q, u64 addr)
|
2018-05-02 19:01:34 +08:00
|
|
|
{
|
|
|
|
struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;
|
|
|
|
|
2018-06-29 15:48:17 +08:00
|
|
|
if (xskq_nb_free(q, q->prod_tail, 1) == 0)
|
2018-06-04 20:05:57 +08:00
|
|
|
return -ENOSPC;
|
|
|
|
|
2019-04-16 20:58:08 +08:00
|
|
|
/* A, matches D */
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 19:57:13 +08:00
|
|
|
ring->desc[q->prod_tail++ & q->ring_mask] = addr;
|
2018-05-02 19:01:34 +08:00
|
|
|
|
|
|
|
/* Order producer and data */
|
2019-04-16 20:58:08 +08:00
|
|
|
smp_wmb(); /* B, matches C */
|
2018-05-02 19:01:34 +08:00
|
|
|
|
|
|
|
WRITE_ONCE(q->ring->producer, q->prod_tail);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-06-04 20:05:57 +08:00
|
|
|
static inline int xskq_produce_addr_lazy(struct xsk_queue *q, u64 addr)
|
|
|
|
{
|
|
|
|
struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;
|
|
|
|
|
|
|
|
if (xskq_nb_free(q, q->prod_head, LAZY_UPDATE_THRESHOLD) == 0)
|
|
|
|
return -ENOSPC;
|
|
|
|
|
2019-04-16 20:58:08 +08:00
|
|
|
/* A, matches D */
|
2018-06-04 20:05:57 +08:00
|
|
|
ring->desc[q->prod_head++ & q->ring_mask] = addr;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void xskq_produce_flush_addr_n(struct xsk_queue *q,
|
|
|
|
u32 nb_entries)
|
|
|
|
{
|
|
|
|
/* Order producer and data */
|
2019-04-16 20:58:08 +08:00
|
|
|
smp_wmb(); /* B, matches C */
|
2018-06-04 20:05:57 +08:00
|
|
|
|
|
|
|
q->prod_tail += nb_entries;
|
|
|
|
WRITE_ONCE(q->ring->producer, q->prod_tail);
|
|
|
|
}
|
|
|
|
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 19:57:13 +08:00
|
|
|
static inline int xskq_reserve_addr(struct xsk_queue *q)
|
2018-05-02 19:01:34 +08:00
|
|
|
{
|
|
|
|
if (xskq_nb_free(q, q->prod_head, 1) == 0)
|
|
|
|
return -ENOSPC;
|
|
|
|
|
2019-04-16 20:58:08 +08:00
|
|
|
/* A, matches D */
|
2018-05-02 19:01:34 +08:00
|
|
|
q->prod_head++;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Rx/Tx queue */
|
|
|
|
|
|
|
|
static inline bool xskq_is_valid_desc(struct xsk_queue *q, struct xdp_desc *d)
|
|
|
|
{
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 19:57:13 +08:00
|
|
|
if (!xskq_is_valid_addr(q, d->addr))
|
2018-05-02 19:01:34 +08:00
|
|
|
return false;
|
|
|
|
|
2019-03-08 15:57:27 +08:00
|
|
|
if (((d->addr + d->len) & q->chunk_mask) != (d->addr & q->chunk_mask) ||
|
|
|
|
d->options) {
|
2018-05-02 19:01:34 +08:00
|
|
|
q->invalid_descs++;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline struct xdp_desc *xskq_validate_desc(struct xsk_queue *q,
|
|
|
|
struct xdp_desc *desc)
|
|
|
|
{
|
|
|
|
while (q->cons_tail != q->cons_head) {
|
|
|
|
struct xdp_rxtx_ring *ring = (struct xdp_rxtx_ring *)q->ring;
|
|
|
|
unsigned int idx = q->cons_tail & q->ring_mask;
|
|
|
|
|
2018-06-04 19:57:11 +08:00
|
|
|
*desc = READ_ONCE(ring->desc[idx]);
|
|
|
|
if (xskq_is_valid_desc(q, desc))
|
2018-05-02 19:01:34 +08:00
|
|
|
return desc;
|
|
|
|
|
|
|
|
q->cons_tail++;
|
|
|
|
}
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline struct xdp_desc *xskq_peek_desc(struct xsk_queue *q,
|
|
|
|
struct xdp_desc *desc)
|
|
|
|
{
|
|
|
|
if (q->cons_tail == q->cons_head) {
|
2019-04-16 20:58:08 +08:00
|
|
|
smp_mb(); /* D, matches A */
|
2018-05-02 19:01:34 +08:00
|
|
|
WRITE_ONCE(q->ring->consumer, q->cons_tail);
|
|
|
|
q->cons_head = q->cons_tail + xskq_nb_avail(q, RX_BATCH_SIZE);
|
|
|
|
|
|
|
|
/* Order consumer and data */
|
2019-04-16 20:58:08 +08:00
|
|
|
smp_rmb(); /* C, matches B */
|
2018-05-02 19:01:34 +08:00
|
|
|
}
|
|
|
|
|
2018-06-04 19:57:11 +08:00
|
|
|
return xskq_validate_desc(q, desc);
|
2018-05-02 19:01:34 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void xskq_discard_desc(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
q->cons_tail++;
|
|
|
|
}
|
2018-05-02 19:01:27 +08:00
|
|
|
|
|
|
|
static inline int xskq_produce_batch_desc(struct xsk_queue *q,
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 19:57:13 +08:00
|
|
|
u64 addr, u32 len)
|
2018-05-02 19:01:27 +08:00
|
|
|
{
|
|
|
|
struct xdp_rxtx_ring *ring = (struct xdp_rxtx_ring *)q->ring;
|
|
|
|
unsigned int idx;
|
|
|
|
|
|
|
|
if (xskq_nb_free(q, q->prod_head, 1) == 0)
|
|
|
|
return -ENOSPC;
|
|
|
|
|
2019-04-16 20:58:08 +08:00
|
|
|
/* A, matches D */
|
2018-05-02 19:01:27 +08:00
|
|
|
idx = (q->prod_head++) & q->ring_mask;
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 19:57:13 +08:00
|
|
|
ring->desc[idx].addr = addr;
|
2018-05-02 19:01:27 +08:00
|
|
|
ring->desc[idx].len = len;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void xskq_produce_flush_desc(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
/* Order producer and data */
|
2019-04-16 20:58:08 +08:00
|
|
|
smp_wmb(); /* B, matches C */
|
2018-05-02 19:01:27 +08:00
|
|
|
|
|
|
|
q->prod_tail = q->prod_head,
|
|
|
|
WRITE_ONCE(q->ring->producer, q->prod_tail);
|
|
|
|
}
|
|
|
|
|
2018-05-02 19:01:34 +08:00
|
|
|
static inline bool xskq_full_desc(struct xsk_queue *q)
|
|
|
|
{
|
2018-05-18 20:00:23 +08:00
|
|
|
return xskq_nb_avail(q, q->nentries) == q->nentries;
|
2018-05-02 19:01:34 +08:00
|
|
|
}
|
|
|
|
|
2018-05-02 19:01:27 +08:00
|
|
|
static inline bool xskq_empty_desc(struct xsk_queue *q)
|
|
|
|
{
|
2018-07-23 17:43:03 +08:00
|
|
|
return xskq_nb_free(q, q->prod_tail, q->nentries) == q->nentries;
|
2018-05-02 19:01:27 +08:00
|
|
|
}
|
|
|
|
|
2018-08-31 19:40:02 +08:00
|
|
|
void xskq_set_umem(struct xsk_queue *q, u64 size, u64 chunk_mask);
|
2018-05-02 19:01:25 +08:00
|
|
|
struct xsk_queue *xskq_create(u32 nentries, bool umem_queue);
|
2018-05-02 19:01:27 +08:00
|
|
|
void xskq_destroy(struct xsk_queue *q_ops);
|
2018-05-02 19:01:24 +08:00
|
|
|
|
2018-09-07 16:18:46 +08:00
|
|
|
/* Executed by the core when the entire UMEM gets freed */
|
|
|
|
void xsk_reuseq_destroy(struct xdp_umem *umem);
|
|
|
|
|
2018-05-02 19:01:24 +08:00
|
|
|
#endif /* _LINUX_XSK_QUEUE_H */
|