1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
|
/*
* Header file for the common RDMA routines used in the server/client example
* program.
*
* Author: Animesh Trivedi
* [email protected]
*
*/
#ifndef RDMA_COMMON_H
#define RDMA_COMMON_H
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <errno.h>
#include <getopt.h>
#include <netdb.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <rdma/rdma_cma.h>
#include <infiniband/verbs.h>
/* Error Macro*/
#define rdma_error(msg, args...) do {\
fprintf(stderr, "%s : %d : ERROR : "msg, __FILE__, __LINE__, ## args);\
}while(0);
#ifdef ACN_RDMA_DEBUG
/* Debug Macro */
#define debug(msg, args...) do {\
printf("DEBUG: "msg, ## args);\
}while(0);
#else
#define debug(msg, args...)
#endif /* ACN_RDMA_DEBUG */
/* Capacity of the completion queue (CQ) */
#define CQ_CAPACITY (16)
/* MAX SGE capacity */
#define MAX_SGE (2)
/* MAX work requests */
#define MAX_WR (8)
/* Default port where the RDMA server is listening */
#define DEFAULT_RDMA_PORT (20886)
/*
* We use attribute so that compiler does not step in and try to pad the structure.
* We use this structure to exchange information between the server and the client.
*
* For details see: http://gcc.gnu.org/onlinedocs/gcc/Type-Attributes.html
*/
struct __attribute((packed)) rdma_buffer_attr {
uint64_t address;
uint32_t length;
union stag {
/* if we send, we call it local stags */
uint32_t local_stag;
/* if we receive, we call it remote stag */
uint32_t remote_stag;
}stag;
};
/* resolves a given destination name to sin_addr */
int get_addr(char *dst, struct sockaddr *addr);
/* prints RDMA buffer info structure */
void show_rdma_buffer_attr(struct rdma_buffer_attr *attr);
/*
* Processes an RDMA connection management (CM) event.
* @echannel: CM event channel where the event is expected.
* @expected_event: Expected event type
* @cm_event: where the event will be stored
*/
int process_rdma_cm_event(struct rdma_event_channel *echannel,
enum rdma_cm_event_type expected_event,
struct rdma_cm_event **cm_event);
/* Allocates an RDMA buffer of size 'length' with permission permission. This
* function will also register the memory and returns a memory region (MR)
* identifier or NULL on error.
* @pd: Protection domain where the buffer should be allocated
* @length: Length of the buffer
* @permission: OR of IBV_ACCESS_* permissions as defined for the enum ibv_access_flags
*/
struct ibv_mr* rdma_buffer_alloc(struct ibv_pd *pd,
uint32_t length,
enum ibv_access_flags permission);
/* Frees a previously allocated RDMA buffer. The buffer must be allocated by
* calling rdma_buffer_alloc();
* @mr: RDMA memory region to free
*/
void rdma_buffer_free(struct ibv_mr *mr);
/* This function registers a previously allocated memory. Returns a memory region
* (MR) identifier or NULL on error.
* @pd: protection domain where to register memory
* @addr: Buffer address
* @length: Length of the buffer
* @permission: OR of IBV_ACCESS_* permissions as defined for the enum ibv_access_flags
*/
struct ibv_mr *rdma_buffer_register(struct ibv_pd *pd,
void *addr,
uint32_t length,
enum ibv_access_flags permission);
/* Deregisters a previously register memory
* @mr: Memory region to deregister
*/
void rdma_buffer_deregister(struct ibv_mr *mr);
/* Processes a work completion (WC) notification.
* @comp_channel: Completion channel where the notifications are expected to arrive
* @wc: Array where to hold the work completion elements
* @max_wc: Maximum number of expected work completion (WC) elements. wc must be
* atleast this size.
*/
int process_work_completion_events(struct ibv_comp_channel *comp_channel,
struct ibv_wc *wc,
int max_wc);
/* prints some details from the cm id */
void show_rdma_cmid(struct rdma_cm_id *id);
#endif /* RDMA_COMMON_H */
|
