Source

oi-build / BROKEN / open-fabrics / librdmacm / patches / base.patch

Full commit
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
diff -r -u /tmp/librdmacm-1.0.14.1/configure librdmacm-1.0.14.1/configure
--- /tmp/librdmacm-1.0.14.1/configure	Tue Feb 15 17:12:14 2011
+++ librdmacm-1.0.14.1/configure	Thu Feb 24 08:39:24 2011
@@ -7625,6 +7625,7 @@
  	esac ;;
       esac
       link_all_deplibs=yes
+      hardcode_libdir_flag_spec=
       ;;
 
     sunos4*)
diff -r -u /tmp/librdmacm-1.0.14.1/Makefile.in librdmacm-1.0.14.1/Makefile.in
--- /tmp/librdmacm-1.0.14.1/Makefile.in	Tue Feb 15 17:12:13 2011
+++ librdmacm-1.0.14.1/Makefile.in	Mon Mar 28 16:49:13 2011
@@ -69,7 +69,7 @@
 	"$(DESTDIR)$(man1dir)" "$(DESTDIR)$(man3dir)" \
 	"$(DESTDIR)$(man7dir)" "$(DESTDIR)$(infinibandincludedir)" \
 	"$(DESTDIR)$(librdmacmincludedir)"
-libLTLIBRARIES_INSTALL = $(INSTALL)
+libLTLIBRARIES_INSTALL = $(INSTALL) -m 755
 LTLIBRARIES = $(lib_LTLIBRARIES)
 src_librdmacm_la_LIBADD =
 am_src_librdmacm_la_OBJECTS = src_librdmacm_la-cma.lo \
@@ -76,7 +76,7 @@
 	src_librdmacm_la-addrinfo.lo src_librdmacm_la-acm.lo
 src_librdmacm_la_OBJECTS = $(am_src_librdmacm_la_OBJECTS)
 am__dirstamp = $(am__leading_dot)dirstamp
-binPROGRAMS_INSTALL = $(INSTALL_PROGRAM)
+binPROGRAMS_INSTALL = $(INSTALL_PROGRAM) -m 755
 PROGRAMS = $(bin_PROGRAMS)
 am_examples_mckey_OBJECTS = mckey.$(OBJEXT)
 examples_mckey_OBJECTS = $(am_examples_mckey_OBJECTS)
diff -r -u /tmp/librdmacm-1.0.14.1/src/cma.h librdmacm-1.0.14.1/src/cma.h
--- /tmp/librdmacm-1.0.14.1/src/cma.h	Mon Oct  4 17:00:18 2010
+++ librdmacm-1.0.14.1/src/cma.h	Fri Feb 11 04:08:57 2011
@@ -40,8 +40,10 @@
 
 #include <stdlib.h>
 #include <errno.h>
+#if !(defined(__SVR4) && defined(__sun))
 #include <endian.h>
 #include <byteswap.h>
+#endif
 
 #include <rdma/rdma_cma.h>
 
@@ -58,14 +60,6 @@
 
 #define PFX "librdmacm: "
 
-#if __BYTE_ORDER == __LITTLE_ENDIAN
-static inline uint64_t htonll(uint64_t x) { return bswap_64(x); }
-static inline uint64_t ntohll(uint64_t x) { return bswap_64(x); }
-#else
-static inline uint64_t htonll(uint64_t x) { return x; }
-static inline uint64_t ntohll(uint64_t x) { return x; }
-#endif
-
 #define min(a, b) (a < b ? a : b)
 
 static inline int ERR(int err)
diff -r -u /tmp/librdmacm-1.0.14.1/src/cma.c librdmacm-1.0.14.1/src/cma.c
--- /tmp/librdmacm-1.0.14.1/src/cma.c	Fri Dec 10 12:05:34 2010
+++ librdmacm-1.0.14.1/src/cma.c	Mon Mar 28 16:44:55 2011
@@ -46,12 +46,15 @@
 #include <poll.h>
 #include <unistd.h>
 #include <pthread.h>
+#if !(defined(__SVR4) && defined(__sun))
 #include <endian.h>
 #include <byteswap.h>
+#endif
 #include <stddef.h>
 #include <netdb.h>
 
 #include "cma.h"
+#include <infiniband/arch.h>
 #include <infiniband/driver.h>
 #include <infiniband/marshall.h>
 #include <rdma/rdma_cma.h>
@@ -354,9 +357,18 @@
 	if (!channel)
 		return NULL;
 
+#if defined(__SVR4) && defined(__sun)
+	channel->fd = open("/dev/infiniband/ofs/rdma_cm", O_RDWR);
+#else
 	channel->fd = open("/dev/infiniband/rdma_cm", O_RDWR);
+#endif
+
 	if (channel->fd < 0) {
+#if defined(__SVR4) && defined(__sun)
+		printf("CMA: unable to open /dev/infiniband/ofs/rdma_cm\n");
+#else
 		printf("CMA: unable to open /dev/infiniband/rdma_cm\n");
+#endif
 		goto err;
 	}
 	return channel;
@@ -1186,6 +1198,10 @@
 	if (ret)
 		return ret;
 
+#if defined(__SVR4) && defined(__sun)
+	qp_init_attr->sq_sig_all |= LIB_RDMACM_QP_BIT;
+#endif
+
 	qp = ibv_create_qp(pd, qp_init_attr);
 	if (!qp) {
 		ret = ERR(ENOMEM);
@@ -1787,6 +1803,9 @@
 
 	CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_ACCEPT, size);
 	cmd->id = id_priv->handle;
+#if defined(__SVR4) && defined(__sun)
+	cmd->conn_param.qp_num = ((id_priv->id).qp)->qp_num;
+#endif
 
 	ret = write(id_priv->id.channel->fd, msg, size);
 	if (ret != size) {
diff -r -u /tmp/librdmacm-1.0.14.1/man/rdma_create_id.3 librdmacm-1.0.14.1/man/rdma_create_id.3
--- /tmp/librdmacm-1.0.14.1/man/rdma_create_id.3	Mon Oct  4 17:00:18 2010
+++ librdmacm-1.0.14.1/man/rdma_create_id.3	Mon Mar 28 03:11:48 2011
@@ -31,9 +31,7 @@
 explicitly binding to a specified RDMA device before communication
 can occur, and most operations are asynchronous in nature.  Asynchronous
 communication events on an rdma_cm_id are reported through the associated
-event channel.  If the channel parameter is NULL, the rdma_cm_id will
-be placed into synchronous operation.  While operating synchronously,
-calls that result in an event will block until the operation completes.
+event channel.
 The event will be returned to the user through the rdma_cm_id structure,
 and be available for access until another rdma_cm call is made.
 .P
diff -r -u /tmp/librdmacm-1.0.14.1/man/rdma_create_qp.3 librdmacm-1.0.14.1/man/rdma_create_qp.3
--- /tmp/librdmacm-1.0.14.1/man/rdma_create_qp.3	Fri Dec 10 12:05:34 2010
+++ librdmacm-1.0.14.1/man/rdma_create_qp.3	Mon Mar 28 03:11:48 2011
@@ -33,8 +33,7 @@
 the rdma_cm_id will be created using a default protection domain.  One
 default protection domain is allocated per RDMA device.
 .P
-The initial QP attributes are specified by the qp_init_attr parameter.  The
-send_cq and recv_cq fields in the ibv_qp_init_attr are optional.  If
+The initial QP attributes are specified by the qp_init_attr parameter.  If
 a send or receive completion queue is not specified, then a CQ will be
 allocated by the rdma_cm for the QP, along with corresponding completion
 channels.  Completion channels and CQ data created by the rdma_cm are
diff -r -u /tmp/librdmacm-1.0.14.1/man/rdma_cm.7 librdmacm-1.0.14.1/man/rdma_cm.7
--- /tmp/librdmacm-1.0.14.1/man/rdma_cm.7	Mon Oct  4 17:00:18 2010
+++ librdmacm-1.0.14.1/man/rdma_cm.7	Mon Mar 28 03:11:47 2011
@@ -19,7 +19,7 @@
 API defined by the libibverbs library.  The libibverbs library provides the
 underlying interfaces needed to send and receive data.
 .P
-The RDMA CM can operate asynchronously or synchronously.  The mode of
+The RDMA CM operates asynchronously.  The mode of
 operation is controlled by the user through the use of the rdma_cm event channel
 parameter in specific calls.  If an event channel is provided, an rdma_cm identifier
 will report its event data (results of connecting, for example), on that channel.
@@ -63,12 +63,7 @@
 .SH "CLIENT OPERATION"
 This section provides a general overview of the basic operation for the active,
 or client, side of communication.  This flow assume asynchronous operation with
-low level call details shown.  For
-synchronous operation, calls to rdma_create_event_channel, rdma_get_cm_event,
-rdma_ack_cm_event, and rdma_destroy_event_channel
-would be eliminated.  Abstracted calls, such as rdma_create_ep encapsulate
-serveral of these calls under a single API.
-Users may also refer to the example applications for
+low level call details shown.  Users may also refer to the example applications for
 code samples.  A general connection flow would be:
 .IP rdma_getaddrinfo
 retrieve address information of the destination
@@ -178,12 +173,9 @@
 rdma_ack_cm_event(3),
 rdma_bind_addr(3),
 rdma_connect(3),
-rdma_create_ep(3),
 rdma_create_event_channel(3),
 rdma_create_id(3),
 rdma_create_qp(3),
-rdma_dereg_mr(3),
-rdma_destroy_ep(3),
 rdma_destroy_event_channel(3),
 rdma_destroy_id(3),
 rdma_destroy_qp(3),
@@ -196,27 +188,11 @@
 rdma_get_dst_port(3),
 rdma_get_local_addr(3),
 rdma_get_peer_addr(3),
-rdma_get_recv_comp(3),
-rdma_get_request(3),
-rdma_get_send_comp(3),
 rdma_get_src_port(3),
 rdma_join_multicast(3),
 rdma_leave_multicast(3),
 rdma_listen(3),
-rdma_migrate_id(3),
 rdma_notify(3),
-rdma_post_read(3)
-rdma_post_readv(3),
-rdma_post_recv(3),
-rdma_post_recvv(3),
-rdma_post_send(3),
-rdma_post_sendv(3),
-rdma_post_ud_send(3),
-rdma_post_write(3),
-rdma_post_writev(3),
-rdma_reg_msgs(3),
-rdma_reg_read(3),
-rdma_reg_write(3),
 rdma_reject(3),
 rdma_resolve_addr(3),
 rdma_resolve_route(3),
diff -r -u /tmp/librdmacm-1.0.14.1/include/infiniband/ib.h librdmacm-1.0.14.1/include/infiniband/ib.h
--- /tmp/librdmacm-1.0.14.1/include/infiniband/ib.h	Mon Oct  4 17:00:18 2010
+++ librdmacm-1.0.14.1/include/infiniband/ib.h	Fri Feb 11 04:08:56 2011
@@ -33,7 +33,11 @@
 #if !defined(_RDMA_IB_H)
 #define _RDMA_IB_H
 
+#if !(defined(__SVR4) && defined(__sun))
 #include <linux/types.h>
+#else
+#include <infiniband/ofa_solaris.h>
+#endif
 #include <string.h>
 
 #ifndef AF_IB
diff -r -u /tmp/librdmacm-1.0.14.1/include/rdma/rdma_cma_abi.h librdmacm-1.0.14.1/include/rdma/rdma_cma_abi.h
--- /tmp/librdmacm-1.0.14.1/include/rdma/rdma_cma_abi.h	Mon Oct  4 17:00:18 2010
+++ librdmacm-1.0.14.1/include/rdma/rdma_cma_abi.h	Fri Feb 11 04:08:48 2011
@@ -104,6 +104,9 @@
 	__u64 response;
 	struct sockaddr_in6 addr;
 	__u32 id;
+#if defined(__SVR4) && defined(__sun)
+	uint32_t	reserved;
+#endif
 };
 
 struct ucma_abi_bind {
@@ -243,6 +246,9 @@
 	__u64 uid;
 	struct sockaddr_in6 addr;
 	__u32 id;
+#if defined(__SVR4) && defined(__sun)
+	uint32_t	reserved;
+#endif
 };
 
 struct ucma_abi_join_mcast {
diff -r -u /tmp/librdmacm-1.0.14.1/examples/udaddy.c librdmacm-1.0.14.1/examples/udaddy.c
--- /tmp/librdmacm-1.0.14.1/examples/udaddy.c	Mon Oct  4 17:00:18 2010
+++ librdmacm-1.0.14.1/examples/udaddy.c	Fri Feb 11 04:08:48 2011
@@ -40,7 +40,9 @@
 #include <netinet/in.h>
 #include <sys/socket.h>
 #include <netdb.h>
+#if !(defined(__SVR4) && defined(__sun))
 #include <byteswap.h>
+#endif
 #include <getopt.h>
 
 #include <rdma/rdma_cma.h>
diff -r -u /tmp/librdmacm-1.0.14.1/examples/mckey.c librdmacm-1.0.14.1/examples/mckey.c
--- /tmp/librdmacm-1.0.14.1/examples/mckey.c	Mon Oct  4 17:00:18 2010
+++ librdmacm-1.0.14.1/examples/mckey.c	Fri Feb 11 04:08:48 2011
@@ -41,7 +41,9 @@
 #include <arpa/inet.h>
 #include <sys/socket.h>
 #include <netdb.h>
+#if !(defined(__SVR4) && defined(__sun))
 #include <byteswap.h>
+#endif
 #include <unistd.h>
 #include <getopt.h>
 
@@ -329,6 +331,16 @@
 
 	while (1) {
 		ret = rdma_get_cm_event(test.channel, &event);
+
+/* 
+ * Solaris returns EBADF if we close the channel while we're waiting
+ * for any events to occur. It is safe to ignore EBADF here.
+ */
+#if defined(__SVR4) && defined(__sun)
+		if (ret && (errno == EBADF))
+			break;
+#endif
+
 		if (ret) {
 			perror("rdma_get_cm_event");
 			break;
@@ -461,6 +473,7 @@
 	return ret;
 }
 
+#if !(defined(__SVR4) && defined(__sun))
 static int get_dst_addr(char *dst, struct sockaddr *addr)
 {
 	struct sockaddr_ib *sib;
@@ -474,6 +487,7 @@
 	inet_pton(AF_INET6, dst, &sib->sib_addr);
 	return 0;
 }
+#endif
 
 static int run(void)
 {
@@ -486,7 +500,12 @@
 			return ret;
 	}
 
+/* Solaris does not yet support family AF_IB */
+#if defined(__SVR4) && defined(__sun)
+	ret = get_addr(dst_addr, (struct sockaddr *) &test.dst_in);
+#else
 	ret = get_dst_addr(dst_addr, (struct sockaddr *) &test.dst_in);
+#endif
 	if (ret)
 		return ret;
 
diff -r -u /tmp/librdmacm-1.0.14.1/examples/cmatose.c librdmacm-1.0.14.1/examples/cmatose.c
--- /tmp/librdmacm-1.0.14.1/examples/cmatose.c	Mon Oct  4 17:00:18 2010
+++ librdmacm-1.0.14.1/examples/cmatose.c	Fri Feb 11 04:08:48 2011
@@ -40,7 +40,9 @@
 #include <netinet/in.h>
 #include <sys/socket.h>
 #include <netdb.h>
+#if !(defined(__SVR4) && defined(__sun))
 #include <byteswap.h>
+#endif
 #include <getopt.h>
 
 #include <rdma/rdma_cma.h>
diff -r -u /tmp/librdmacm-1.0.14.1/examples/rping.c librdmacm-1.0.14.1/examples/rping.c
--- /tmp/librdmacm-1.0.14.1/examples/rping.c	Tue Feb 15 17:10:48 2011
+++ librdmacm-1.0.14.1/examples/rping.c	Wed May 25 14:46:30 2011
@@ -40,11 +40,17 @@
 #include <netinet/in.h>
 #include <sys/socket.h>
 #include <netdb.h>
+#if !(defined(__SVR4) && defined(__sun))
 #include <byteswap.h>
+#endif
 #include <semaphore.h>
 #include <arpa/inet.h>
 #include <pthread.h>
 #include <inttypes.h>
+#if defined(__SVR4) && defined(__sun)
+#include <unistd.h>
+#include <libgen.h>
+#endif
 
 #include <rdma/rdma_cma.h>
 #include <infiniband/arch.h>
@@ -85,6 +91,13 @@
 	ERROR
 };
 
+enum disconnect_state {
+	DISCONNECT_NONE,
+	CALLING_DISCONNECT = 1,
+	DISCONNECT_CALLED,
+	DISCONNECT_DONE
+};
+
 struct rping_rdma_info {
 	uint64_t buf;
 	uint32_t rkey;
@@ -143,6 +156,9 @@
 	enum test_state state;		/* used for cond/signalling */
 	sem_t sem;
 
+	enum disconnect_state  dis_state;
+	sem_t dis_sem;
+
 	struct sockaddr_storage sin;
 	uint16_t port;			/* dst port in NBO */
 	int verbose;			/* verbose logging */
@@ -218,6 +234,8 @@
 		fprintf(stderr, "%s DISCONNECT EVENT...\n",
 			cb->server ? "server" : "client");
 		sem_post(&cb->sem);
+		cb->dis_state = DISCONNECT_DONE;
+		sem_post(&cb->dis_sem);
 		break;
 
 	case RDMA_CM_EVENT_DEVICE_REMOVAL:
@@ -285,6 +303,29 @@
 					"cq completion failed status %d\n",
 					wc.status);
 				ret = -1;
+			} else {
+				/*
+				 * FLUSH Error can be polled before RDMA-CM
+				 * DISCONNECT is notified. Ensure that cb_state
+				 * is set appropriately in such a case.
+				 * sleep for sometime if Disconnect has not
+				 * been called. The FLUSH WR can be because
+				 * the remote end initiated the disconnect.
+				 */
+				if (cb->dis_state == DISCONNECT_NONE)
+					sleep(2);
+
+				if (cb->dis_state == DISCONNECT_DONE)
+					return (0);
+
+				/* Wait if disconnect is called. */
+				if (cb->dis_state == DISCONNECT_CALLED) {
+					sem_wait(&cb->dis_sem);
+					if (cb->dis_state == DISCONNECT_DONE)
+						return (0);
+					else
+						goto error;
+				}
 			}
 			goto error;
 		}
@@ -571,9 +612,15 @@
 
 	while (1) {
 		ret = rdma_get_cm_event(cb->cm_channel, &event);
-		if (ret) {
+		/*
+		 * If the retry of read() syscall returned EBADF, as the
+		 * file was closed on process exit. Ignore this error.
+		 */
+		if (ret && errno != EBADF) {
 			perror("rdma_get_cm_event");
 			exit(ret);
+		} else if (ret && errno == EBADF) {
+			exit(0);
 		}
 		ret = rping_cma_event_handler(event->id, event);
 		rdma_ack_cm_event(event);
@@ -595,8 +642,14 @@
 		pthread_testcancel();
 
 		ret = ibv_get_cq_event(cb->channel, &ev_cq, &ev_ctx);
-		if (ret) {
+		/*
+		 * If the retry of write() syscall returned EBADF, as the
+		 * file was closed on process exit. Ignore this error.
+		 */
+		if (ret && errno != EBADF) {
 			fprintf(stderr, "Failed to get cq event!\n");
+			 pthread_exit(NULL);
+		} else if (ret && errno == EBADF) {
 			pthread_exit(NULL);
 		}
 		if (ev_cq != cb->cq) {
@@ -801,11 +854,13 @@
 	}
 
 	rping_test_server(cb);
+	cb->dis_state = CALLING_DISCONNECT;
+	sem_post(&cb->dis_sem);
 	rdma_disconnect(cb->child_cm_id);
-	rping_free_buffers(cb);
-	rping_free_qp(cb);
 	pthread_cancel(cb->cqthread);
 	pthread_join(cb->cqthread, NULL);
+	rping_free_buffers(cb);
+	rping_free_qp(cb);
 	rdma_destroy_id(cb->child_cm_id);
 	free_cb(cb);
 	return NULL;
@@ -889,6 +944,8 @@
 	}
 
 	rping_test_server(cb);
+	cb->dis_state = CALLING_DISCONNECT;
+	sem_post(&cb->dis_sem);
 	rdma_disconnect(cb->child_cm_id);
 	rdma_destroy_id(cb->child_cm_id);
 err2:
@@ -1056,6 +1113,8 @@
 	}
 
 	rping_test_client(cb);
+	cb->dis_state = CALLING_DISCONNECT;
+	sem_post(&cb->dis_sem);
 	rdma_disconnect(cb->cm_id);
 err2:
 	rping_free_buffers(cb);
@@ -1123,6 +1182,7 @@
 	cb->sin.ss_family = PF_INET;
 	cb->port = htons(7174);
 	sem_init(&cb->sem, 0, 0);
+	sem_init(&cb->dis_sem, 0, 0);
 
 	opterr = 0;
 	while ((op=getopt(argc, argv, "a:Pp:C:S:t:scvVd")) != -1) {