Functions to initialize and interface directly with the PCIe device. More...

#include "pcie.h"
#include <arpa/inet.h>
#include <enso/config.h>
#include <enso/consts.h>
#include <enso/helpers.h>
#include <immintrin.h>
#include <sched.h>
#include <string.h>
#include <sys/mman.h>
#include <time.h>
#include <unistd.h>
#include <algorithm>
#include <cassert>
#include <cerrno>
#include <cstdint>
#include <cstdlib>
#include <ctime>
#include <iomanip>
#include <iostream>
#include <limits>
#include <stdexcept>
#include <dev_backend.h>

Functions
int	enso::notification_buf_init (uint32_t bdf, int32_t bar, struct NotificationBufPair *notification_buf_pair, const std::string &huge_page_prefix)
	Initializes the notification buffer pair.

int	enso::enso_pipe_init (struct RxEnsoPipeInternal enso_pipe, struct NotificationBufPair notification_buf_pair, bool fallback)
	Initializes an Enso Pipe.

int	enso::dma_init (struct NotificationBufPair notification_buf_pair, struct RxEnsoPipeInternal enso_pipe, uint32_t bdf, int32_t bar, const std::string &huge_page_prefix, bool fallback)
	Initializes an enso pipe and the notification buffer if needed.

uint16_t	enso::get_new_tails (struct NotificationBufPair *notification_buf_pair)
	Gets latest tails for the pipes associated with the given notification buffer.

uint32_t	enso::get_next_batch_from_queue (struct RxEnsoPipeInternal enso_pipe, struct NotificationBufPair notification_buf_pair, void **buf)
	Gets the next batch of data from the given Enso Pipe.

uint32_t	enso::peek_next_batch_from_queue (struct RxEnsoPipeInternal enso_pipe, struct NotificationBufPair notification_buf_pair, void **buf)
	Gets the next batch of data from the given Enso Pipe without consuming it. So the next call to `get_next_batch_from_queue` or to `peek_next_batch_from_queue` will return the same data.

int32_t	enso::get_next_enso_pipe_id (struct NotificationBufPair *notification_buf_pair)
	Get next Enso Pipe with pending data.

uint32_t	enso::get_next_batch (struct NotificationBufPair notification_buf_pair, struct SocketInternal socket_entries, int enso_pipe_id, void *buf)
	Get next batch of data from the next available Enso Pipe.

void	enso::advance_pipe (struct RxEnsoPipeInternal *enso_pipe, size_t len)
	Frees the next `len` bytes in the buffer associated with the `socket_entry` socket. If `len` is greater than the number of allocated bytes in the buffer, the behavior is undefined.

void	enso::fully_advance_pipe (struct RxEnsoPipeInternal *enso_pipe)
	Frees all the received bytes in the buffer associated with the `socket_entry` socket.

void	enso::prefetch_pipe (struct RxEnsoPipeInternal *enso_pipe)
	Prefetches a given Enso Pipe.

uint32_t	enso::send_to_queue (struct NotificationBufPair *notification_buf_pair, uint64_t phys_addr, uint32_t len)
	Sends data through a given queue.

uint32_t	enso::get_unreported_completions (struct NotificationBufPair *notification_buf_pair)
	Returns the number of transmission requests that were completed since the last call to this function.

void	enso::update_tx_head (struct NotificationBufPair *notification_buf_pair)
	Updates the tx head and the number of TX completions.

int	enso::send_config (struct NotificationBufPair notification_buf_pair, struct TxNotification config_notification)
	Sends configuration to the NIC.

int	enso::get_nb_fallback_queues (struct NotificationBufPair *notification_buf_pair)
	Get number of fallback queues currently in use.

int	enso::set_round_robin_status (struct NotificationBufPair *notification_buf_pair, bool round_robin)
	Sets the round robin status for the device.

int	enso::get_round_robin_status (struct NotificationBufPair *notification_buf_pair)
	Gets the round robin status for the device.

uint64_t	enso::get_dev_addr_from_virt_addr (struct NotificationBufPair notification_buf_pair, void virt_addr)
	Converts an address in the application's virtual address space to an address that can be used by the device (typically a physical address).

void	enso::notification_buf_free (struct NotificationBufPair *notification_buf_pair)
	Frees the notification buffer pair.

void	enso::enso_pipe_free (struct NotificationBufPair notification_buf_pair, struct RxEnsoPipeInternal enso_pipe, enso_pipe_id_t enso_pipe_id)
	Frees the Enso Pipe.

int	enso::dma_finish (struct SocketInternal *socket_entry)
	Frees the notification buffer and all pipes.

uint32_t	enso::get_enso_pipe_id_from_socket (struct SocketInternal *socket_entry)
	Gets the Enso Pipe ID associated with a given socket.

void	enso::print_stats (struct SocketInternal *socket_entry, bool print_global)
	Prints statistics for a given socket.

Detailed Description

Functions to initialize and interface directly with the PCIe device.

Author: Hugo Sadok sadok.nosp@m.@cmu.nosp@m..edu

Definition in file pcie.cpp.

Function Documentation

◆ advance_pipe()

void enso::advance_pipe	(	struct RxEnsoPipeInternal *	enso_pipe,
		size_t	len
	)

Frees the next len bytes in the buffer associated with the socket_entry socket. If len is greater than the number of allocated bytes in the buffer, the behavior is undefined.

Parameters

enso_pipe	Enso pipe to advance.
len	Number of bytes to free.

Definition at line 446 of file pcie.cpp.

◆ dma_finish()

int enso::dma_finish ( struct SocketInternal * socket_entry )

Frees the notification buffer and all pipes.

Deprecated:: This function is deprecated and will be removed in the future.

Definition at line 684 of file pcie.cpp.

◆ dma_init()

int enso::dma_init	(	struct NotificationBufPair *	notification_buf_pair,
		struct RxEnsoPipeInternal *	enso_pipe,
		uint32_t	bdf,
		int32_t	bar,
		const std::string &	huge_page_prefix,
		bool	fallback
	)

Initializes an enso pipe and the notification buffer if needed.

Deprecated:: This function is deprecated and will be removed in the future.

Definition at line 285 of file pcie.cpp.

◆ enso_pipe_free()

void enso::enso_pipe_free	(	struct NotificationBufPair *	notification_buf_pair,
		struct RxEnsoPipeInternal *	enso_pipe,
		enso_pipe_id_t	enso_pipe_id
	)

Frees the Enso Pipe.

Parameters

notification_buf_pair	Notification buffer pair to use.
enso_pipe	Enso Pipe to free.
enso_pipe_id	Hardware ID of the Enso Pipe to free.

Definition at line 661 of file pcie.cpp.

◆ enso_pipe_init()

int enso::enso_pipe_init	(	struct RxEnsoPipeInternal *	enso_pipe,
		struct NotificationBufPair *	notification_buf_pair,
		bool	fallback
	)

Initializes an Enso Pipe.

Parameters

enso_pipe	Enso Pipe to initialize.
notification_buf_pair	Notification buffer pair to use.
fallback	Whether the queues is a fallback queue or not.

Returns: Pipe ID on success, -1 on failure.

Definition at line 214 of file pcie.cpp.

◆ fully_advance_pipe()

void enso::fully_advance_pipe ( struct RxEnsoPipeInternal * enso_pipe )

Frees all the received bytes in the buffer associated with the socket_entry socket.

Parameters

enso_pipe Enso pipe to advance.

Definition at line 455 of file pcie.cpp.

◆ get_dev_addr_from_virt_addr()

uint64_t enso::get_dev_addr_from_virt_addr	(	struct NotificationBufPair *	notification_buf_pair,
		void *	virt_addr
	)

Converts an address in the application's virtual address space to an address that can be used by the device (typically a physical address).

Parameters

notification_buf_pair	Notification buffer pair to use.
virt_addr	Virtual address to convert.

Returns: Converted address or 0 if the address cannot be translated.

Definition at line 627 of file pcie.cpp.

◆ get_enso_pipe_id_from_socket()

uint32_t enso::get_enso_pipe_id_from_socket ( struct SocketInternal * socket_entry )

Gets the Enso Pipe ID associated with a given socket.

Deprecated:: This function is deprecated and will be removed in the future.

Definition at line 707 of file pcie.cpp.

◆ get_nb_fallback_queues()

int enso::get_nb_fallback_queues ( struct NotificationBufPair * notification_buf_pair )

Get number of fallback queues currently in use.

Parameters

notification_buf_pair Notification buffer pair to use.

Returns: Number of fallback queues currently in use or -1 on failure.

Definition at line 608 of file pcie.cpp.

◆ get_new_tails()

uint16_t enso::get_new_tails ( struct NotificationBufPair * notification_buf_pair )

Gets latest tails for the pipes associated with the given notification buffer.

Parameters

notification_buf_pair Notification buffer to get data from.

Returns: Number of notifications received.

Definition at line 353 of file pcie.cpp.

◆ get_next_batch()

uint32_t enso::get_next_batch	(	struct NotificationBufPair *	notification_buf_pair,
		struct SocketInternal *	socket_entries,
		int *	enso_pipe_id,
		void **	buf
	)

Get next batch of data from the next available Enso Pipe.

Parameters

notification_buf_pair	Notification buffer to get data from.
socket_entries	Array of socket entries.
enso_pipe_id	ID of the Enso Pipe that the data came from.
buf	Pointer to the buffer where the data will be stored.

Returns: Number of bytes received.

Definition at line 429 of file pcie.cpp.

◆ get_next_batch_from_queue()

uint32_t enso::get_next_batch_from_queue	(	struct RxEnsoPipeInternal *	enso_pipe,
		struct NotificationBufPair *	notification_buf_pair,
		void **	buf
	)

Gets the next batch of data from the given Enso Pipe.

Parameters

enso_pipe	Enso Pipe to get data from.
notification_buf_pair	Notification buffer to get data from.
buf	Pointer to the buffer where the data will be stored, it will be updated to point to the next available data.

Returns: Number of bytes received.

Definition at line 385 of file pcie.cpp.

◆ get_next_enso_pipe_id()

int32_t enso::get_next_enso_pipe_id ( struct NotificationBufPair * notification_buf_pair )

Get next Enso Pipe with pending data.

Parameters

notification_buf_pair Notification buffer to get data from.

Returns: ID for the next Enso Pipe that has data available, or -1 if no Enso Pipe has data.

Definition at line 423 of file pcie.cpp.

◆ get_round_robin_status()

int enso::get_round_robin_status ( struct NotificationBufPair * notification_buf_pair )

Gets the round robin status for the device.

Parameters

notification_buf_pair Notification buffer pair to use.

Returns: 0 if round robin is disabled, 1 if round robin is enabled, -1 on failure.

Definition at line 621 of file pcie.cpp.

◆ get_unreported_completions()

uint32_t enso::get_unreported_completions ( struct NotificationBufPair * notification_buf_pair )

Returns the number of transmission requests that were completed since the last call to this function.

Since transmissions are always completed in order, one can figure out which transmissions were completed by keeping track of all the calls to send_to_queue. There can be only up to kMaxPendingTxRequests requests completed between two calls to send_to_queue. However, if send is called multiple times, without calling get_unreported_completions the number of completed requests can surpass kMaxPendingTxRequests.

Parameters

notification_buf_pair Notification buffer to get completions from.

Returns: number of transmission requests that were completed since the last call to this function.

Definition at line 520 of file pcie.cpp.

◆ notification_buf_free()

void enso::notification_buf_free ( struct NotificationBufPair * notification_buf_pair )

Frees the notification buffer pair.

Parameters

notification_buf_pair Notification buffer pair to free.

Definition at line 635 of file pcie.cpp.

◆ notification_buf_init()

int enso::notification_buf_init	(	uint32_t	bdf,
		int32_t	bar,
		struct NotificationBufPair *	notification_buf_pair,
		const std::string &	huge_page_prefix
	)

Initializes the notification buffer pair.

Parameters

bdf	BDF of the PCIe device to use.
bar	PCIe BAR to use (set to -1 to automatically select one).
notification_buf_pair	Notification buffer pair to initialize.
huge_page_prefix	File prefix to use when allocating the huge pages.

Returns: 0 on success, -1 on failure.

Definition at line 75 of file pcie.cpp.

◆ peek_next_batch_from_queue()

uint32_t enso::peek_next_batch_from_queue	(	struct RxEnsoPipeInternal *	enso_pipe,
		struct NotificationBufPair *	notification_buf_pair,
		void **	buf
	)

Gets the next batch of data from the given Enso Pipe without consuming it. So the next call to get_next_batch_from_queue or to peek_next_batch_from_queue will return the same data.

Parameters

enso_pipe	Enso Pipe to get data from.
notification_buf_pair	Notification buffer to get data from.
buf	Pointer to the buffer where the data will be stored, it will be updated to point to the next available data.

Returns: Number of bytes received.

Definition at line 391 of file pcie.cpp.

◆ prefetch_pipe()

void enso::prefetch_pipe ( struct RxEnsoPipeInternal * enso_pipe )

Prefetches a given Enso Pipe.

Parameters

enso_pipe Enso pipe to prefetch.

Definition at line 460 of file pcie.cpp.

◆ print_stats()

void enso::print_stats	(	struct SocketInternal *	socket_entry,
		bool	print_global
	)

Prints statistics for a given socket.

Deprecated:: This function is deprecated and will be removed in the future.

Definition at line 711 of file pcie.cpp.

◆ send_config()

int enso::send_config	(	struct NotificationBufPair *	notification_buf_pair,
		struct TxNotification *	config_notification
	)

Sends configuration to the NIC.

Parameters

notification_buf_pair	The notification buffer pair to send the configuration through.
config_notification	The configuration notification to send. Must be a config notification, i.e., signal >= 2.

Returns: 0 on success, -1 on failure.

Definition at line 569 of file pcie.cpp.

◆ send_to_queue()

uint32_t enso::send_to_queue	(	struct NotificationBufPair *	notification_buf_pair,
		uint64_t	phys_addr,
		uint32_t	len
	)

Sends data through a given queue.

This function returns as soon as a transmission requests has been enqueued to the TX notification buffer. That means that it is not safe to modify or deallocate the buffer pointed by phys_addr right after it returns. Instead, the caller must use get_unreported_completions to figure out when the transmission is complete.

This function currently blocks if there is not enough space in the notification buffer.

Parameters

notification_buf_pair	Notification buffer to send data through.
phys_addr	Physical memory address of the data to be sent.
len	Length, in bytes, of the data.

Returns: number of bytes sent.

Definition at line 515 of file pcie.cpp.

◆ set_round_robin_status()

int enso::set_round_robin_status	(	struct NotificationBufPair *	notification_buf_pair,
		bool	round_robin
	)

Sets the round robin status for the device.

Parameters

notification_buf_pair	Notification buffer pair to use.
round_robin	Whether to enable or disable round robin.

Returns: 0 on success, -1 on failure.

Definition at line 614 of file pcie.cpp.

◆ update_tx_head()

void enso::update_tx_head ( struct NotificationBufPair * notification_buf_pair )

Updates the tx head and the number of TX completions.

Parameters

notification_buf_pair Notification buffer to be updated.

Definition at line 530 of file pcie.cpp.