libnpfs/rdmasrv.c

   1 #include <stdlib.h>
   2 #include <string.h>
   3 #include <stdio.h>
   4 #include <unistd.h>
   5 #include <sys/socket.h>
   6 #include <netinet/in.h>
   7 #include <errno.h>
   8 #include "npfs.h"
   9 #include "npfsimpl.h"
  10 #include <infiniband/verbs.h>
  11 #include <rdma/rdma_cma.h>
  12
  13 typedef struct Rdmasrv Rdmasrv;
  14 struct Rdmasrv {
  15         struct rdma_cm_id *listen_id;
  16         struct rdma_event_channel *event_channel;
  17         struct sockaddr_in addr;
  18         int shutdown;
  19         pthread_t thread;
  20 };
  21
  22 extern Nptrans *np_rdmatrans_create(struct rdma_cm_id *cmid, int q_depth, int msize);
  23
  24 static void rdmasrv_start(Npsrv *srv);
  25 static void rdmasrv_shutdown(Npsrv *srv);
  26 static void rdmasrv_destroy(Npsrv *srv);
  27 static void *rdmasrv_listenproc(void *a);
  28
  29 Npsrv*
  30 np_rdmasrv_create(int nwthreads, int *port)
  31 {
  32         Npsrv *srv;
  33         Rdmasrv *rdma;
  34         int ret;
  35
  36         rdma = malloc(sizeof(struct Rdmasrv));
  37         if (!rdma)
  38                 return NULL;
  39
  40         rdma->event_channel = rdma_create_event_channel();
  41         if (!rdma->event_channel) {
  42                 np_werror("Can't create event channel", EIO);
  43                 goto error;
  44         }
  45
  46         ret = rdma_create_id(rdma->event_channel, &rdma->listen_id,
  47                              NULL, RDMA_PS_TCP);
  48         if (ret) {
  49                 np_werror("Can't create an id", ret);
  50                 goto error;
  51         }
  52
  53         rdma->addr.sin_family = AF_INET;
  54         rdma->addr.sin_port = htons(*port);
  55         rdma->addr.sin_addr.s_addr = htonl(INADDR_ANY);
  56         ret = rdma_bind_addr(rdma->listen_id, (struct sockaddr *)&rdma->addr);
  57         if (ret) {
  58                 np_werror("Can't bind", ret);
  59                 goto error;
  60         }
  61
  62         srv = np_srv_create(nwthreads);
  63         srv->srvaux = rdma;
  64         srv->start = rdmasrv_start;
  65         srv->shutdown = rdmasrv_shutdown;
  66         srv->destroy = rdmasrv_destroy;
  67
  68         return srv;
  69
  70  error:
  71         free(rdma);
  72         return NULL;
  73 }
  74
  75 static void
  76 rdmasrv_start(Npsrv *srv)
  77 {
  78         int n;
  79         Rdmasrv *rdma;
  80
  81         rdma = srv->srvaux;
  82         n = rdma_listen(rdma->listen_id, 1);
  83         if (n < 0) {
  84                 np_uerror(n);
  85                 return;
  86         }
  87
  88         n = pthread_create(&rdma->thread, NULL, rdmasrv_listenproc, srv);
  89         if (n)
  90                 np_uerror(n);
  91 }
  92
  93 static void
  94 rdmasrv_shutdown(Npsrv *srv)
  95 {
  96         Rdmasrv *rdma;
  97
  98         rdma = srv->srvaux;
  99         rdma->shutdown = 1;
 100         if (rdma->listen_id)
 101                 rdma_destroy_id(rdma->listen_id);
 102
 103         rdma->listen_id = NULL;
 104 }
 105
 106 static void
 107 rdmasrv_destroy(Npsrv *srv)
 108 {
 109         Rdmasrv *rdma;
 110         void *ret;
 111
 112         rdma = srv->srvaux;
 113         rdmasrv_shutdown(srv);
 114         pthread_join(rdma->thread, &ret);
 115         free(rdma);
 116         srv->srvaux = NULL;
 117 }
 118
 119 static void *
 120 rdmasrv_listenproc(void *a)
 121 {
 122         int ret;
 123         Npsrv *srv;
 124         Npconn *conn;
 125         Nptrans *trans;
 126         Rdmasrv *rdma;
 127         struct rdma_cm_event *event;
 128         struct rdma_cm_id *cmid;
 129         enum rdma_cm_event_type etype;
 130
 131         srv = a;
 132         rdma = srv->srvaux;
 133         while (!rdma->shutdown) {
 134                 ret = rdma_get_cm_event(rdma->event_channel, &event);
 135                 if (ret)
 136                         goto error;
 137
 138                 cmid = (struct rdma_cm_id *)event->id;
 139                 etype = event->event;
 140                 rdma_ack_cm_event(event);
 141
 142                 switch (etype) {
 143                 case RDMA_CM_EVENT_CONNECT_REQUEST:
 144                         printf("Connection request\n");
 145                         trans = np_rdmatrans_create(cmid, srv->nwthread, srv->msize);
 146                         if (trans) {
 147                                 conn = np_conn_create(srv, trans);
 148                                 cmid->context = conn;
 149                                 np_srv_add_conn(srv, conn);
 150                         }
 151                         break;
 152
 153                 case RDMA_CM_EVENT_ESTABLISHED:
 154                         printf("Connection established\n");
 155                         break;
 156
 157                 case RDMA_CM_EVENT_DISCONNECTED:
 158                         printf("Connection shutting down\n");
 159                         conn = cmid->context;
 160                         np_conn_shutdown(conn);
 161                         break;
 162
 163                 default:
 164                         fprintf(stderr, "event %d received waiting for a connect request\n",
 165                                 etype);
 166                 }
 167         }
 168         return NULL;
 169
 170  error:
 171         fprintf(stderr, "shutting down the server with error %d\n", ret);
 172         return 0;
 173 }