ctdb/server/ctdb_server.c

   1 /*
   2    ctdb main protocol code
   3
   4    Copyright (C) Andrew Tridgell  2006
   5
   6    This program is free software; you can redistribute it and/or modify
   7    it under the terms of the GNU General Public License as published by
   8    the Free Software Foundation; either version 3 of the License, or
   9    (at your option) any later version.
  10
  11    This program is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14    GNU General Public License for more details.
  15
  16    You should have received a copy of the GNU General Public License
  17    along with this program; if not, see <http://www.gnu.org/licenses/>.
  18 */
  19
  20 #include "replace.h"
  21 #include "system/network.h"
  22 #include "system/filesys.h"
  23
  24 #include <talloc.h>
  25 #include <tevent.h>
  26
  27 #include "lib/util/dlinklist.h"
  28 #include "lib/util/debug.h"
  29 #include "lib/util/samba_util.h"
  30
  31 #include "ctdb_private.h"
  32 #include "ctdb_client.h"
  33
  34 #include "protocol/protocol.h"
  35
  36 #include "common/common.h"
  37 #include "common/logging.h"
  38
  39 #include "conf/node.h"
  40
  41 /*
  42   choose the transport we will use
  43 */
  44 int ctdb_set_transport(struct ctdb_context *ctdb, const char *transport)
  45 {
  46         ctdb->transport = talloc_strdup(ctdb, transport);
  47         if (ctdb->transport == NULL) {
  48                 DBG_ERR("Memory allocation error\n");
  49                 return -1;
  50         }
  51
  52         return 0;
  53 }
  54
  55 /* Return the node structure for nodeip, NULL if nodeip is invalid */
  56 struct ctdb_node *ctdb_ip_to_node(struct ctdb_context *ctdb,
  57                                   const ctdb_sock_addr *nodeip)
  58 {
  59         unsigned int nodeid;
  60
  61         for (nodeid=0;nodeid<ctdb->num_nodes;nodeid++) {
  62                 if (ctdb->nodes[nodeid]->flags & NODE_FLAGS_DELETED) {
  63                         continue;
  64                 }
  65                 if (ctdb_same_ip(&ctdb->nodes[nodeid]->address, nodeip)) {
  66                         return ctdb->nodes[nodeid];
  67                 }
  68         }
  69
  70         return NULL;
  71 }
  72
  73 /* Return the PNN for nodeip, CTDB_UNKNOWN_PNN if nodeip is invalid */
  74 uint32_t ctdb_ip_to_pnn(struct ctdb_context *ctdb,
  75                         const ctdb_sock_addr *nodeip)
  76 {
  77         struct ctdb_node *node;
  78
  79         node = ctdb_ip_to_node(ctdb, nodeip);
  80         if (node == NULL) {
  81                 return CTDB_UNKNOWN_PNN;
  82         }
  83
  84         return node->pnn;
  85 }
  86
  87 /* Load a nodes list file into a nodes array */
  88 static int convert_node_map_to_list(struct ctdb_context *ctdb,
  89                                     TALLOC_CTX *mem_ctx,
  90                                     struct ctdb_node_map *node_map,
  91                                     struct ctdb_node ***nodes,
  92                                     uint32_t *num_nodes)
  93 {
  94         unsigned int i;
  95
  96         *nodes = talloc_zero_array(mem_ctx,
  97                                         struct ctdb_node *, node_map->num);
  98         if (*nodes == NULL) {
  99                 DBG_ERR("Memory allocation error\n");
 100                 return -1;
 101         }
 102         *num_nodes = node_map->num;
 103
 104         for (i = 0; i < node_map->num; i++) {
 105                 struct ctdb_node *node;
 106
 107                 node = talloc_zero(*nodes, struct ctdb_node);
 108                 if (node == NULL) {
 109                         DBG_ERR("Memory allocation error\n");
 110                         TALLOC_FREE(*nodes);
 111                         return -1;
 112                 }
 113                 (*nodes)[i] = node;
 114
 115                 node->address = node_map->node[i].addr;
 116                 node->name = talloc_asprintf(node, "%s:%u",
 117                                              ctdb_addr_to_str(&node->address),
 118                                              ctdb_addr_to_port(&node->address));
 119                 if (node->name == NULL) {
 120                         DBG_ERR("Memory allocation error\n");
 121                         TALLOC_FREE(*nodes);
 122                         return -1;
 123                 }
 124
 125                 node->flags = node_map->node[i].flags;
 126                 if (!(node->flags & NODE_FLAGS_DELETED)) {
 127                         node->flags = NODE_FLAGS_UNHEALTHY;
 128                 }
 129                 node->flags |= NODE_FLAGS_DISCONNECTED;
 130
 131                 node->pnn = i;
 132                 node->ctdb = ctdb;
 133                 node->dead_count = 0;
 134         }
 135
 136         return 0;
 137 }
 138
 139 /* Load the nodes list from a file or sub-processes' stdout  */
 140 void ctdb_load_nodes(struct ctdb_context *ctdb)
 141 {
 142         struct ctdb_node_map *node_map;
 143         int ret;
 144
 145         node_map = ctdb_read_nodes(ctdb, ctdb->nodes_source);
 146         if (node_map == NULL) {
 147                 goto fail;
 148         }
 149
 150         TALLOC_FREE(ctdb->nodes);
 151         ret = convert_node_map_to_list(ctdb, ctdb, node_map,
 152                                        &ctdb->nodes, &ctdb->num_nodes);
 153         if (ret == -1) {
 154                 goto fail;
 155         }
 156
 157         talloc_free(node_map);
 158         return;
 159
 160 fail:
 161         DEBUG(DEBUG_ERR, ("Failed to load nodes \"%s\"\n",
 162                           ctdb->nodes_source));
 163         talloc_free(node_map);
 164         exit(1);
 165 }
 166
 167 /*
 168   setup the local node address
 169 */
 170 int ctdb_set_address(struct ctdb_context *ctdb, const char *address)
 171 {
 172         bool ok;
 173
 174         ctdb->address = talloc(ctdb, ctdb_sock_addr);
 175         if (ctdb->address == NULL) {
 176                 DBG_ERR("Memory allocation error\n");
 177                 return -1;
 178         }
 179
 180         ok = ctdb_parse_node_address(address, ctdb->address);
 181         if (!ok) {
 182                 DBG_ERR("Failed to parse node address\n");
 183                 TALLOC_FREE(ctdb->address);
 184                 return -1;
 185         }
 186
 187         ctdb->name = talloc_asprintf(ctdb, "%s:%u",
 188                                      ctdb_addr_to_str(ctdb->address),
 189                                      ctdb_addr_to_port(ctdb->address));
 190         if (ctdb->name == NULL) {
 191                 DBG_ERR("Memory allocation error\n");
 192                 TALLOC_FREE(ctdb->address);
 193                 return -1;
 194         }
 195
 196         return 0;
 197 }
 198
 199
 200 /*
 201   return the number of active nodes
 202 */
 203 uint32_t ctdb_get_num_active_nodes(struct ctdb_context *ctdb)
 204 {
 205         unsigned int i;
 206         uint32_t count=0;
 207         for (i=0; i < ctdb->num_nodes; i++) {
 208                 if (!(ctdb->nodes[i]->flags & NODE_FLAGS_INACTIVE)) {
 209                         count++;
 210                 }
 211         }
 212         return count;
 213 }
 214
 215
 216 /*
 217   called when we need to process a packet. This can be a requeued packet
 218   after a lockwait, or a real packet from another node
 219 */
 220 void ctdb_input_pkt(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
 221 {
 222         TALLOC_CTX *tmp_ctx;
 223
 224         /* place the packet as a child of the tmp_ctx. We then use
 225            talloc_free() below to free it. If any of the calls want
 226            to keep it, then they will steal it somewhere else, and the
 227            talloc_free() will only free the tmp_ctx */
 228         tmp_ctx = talloc_new(ctdb);
 229         talloc_steal(tmp_ctx, hdr);
 230
 231         DEBUG(DEBUG_DEBUG,(__location__ " ctdb request %u of type %u length %u from "
 232                  "node %u to %u\n", hdr->reqid, hdr->operation, hdr->length,
 233                  hdr->srcnode, hdr->destnode));
 234
 235         switch (hdr->operation) {
 236         case CTDB_REQ_CALL:
 237         case CTDB_REPLY_CALL:
 238         case CTDB_REQ_DMASTER:
 239         case CTDB_REPLY_DMASTER:
 240                 /* we don't allow these calls when banned */
 241                 if (ctdb->nodes[ctdb->pnn]->flags & NODE_FLAGS_BANNED) {
 242                         DEBUG(DEBUG_DEBUG,(__location__ " ctdb operation %u"
 243                                 " request %u"
 244                                 " length %u from node %u to %u while node"
 245                                 " is banned\n",
 246                                  hdr->operation, hdr->reqid,
 247                                  hdr->length,
 248                                  hdr->srcnode, hdr->destnode));
 249                         goto done;
 250                 }
 251
 252                 /* for ctdb_call inter-node operations verify that the
 253                    remote node that sent us the call is running in the
 254                    same generation instance as this node
 255                 */
 256                 if (ctdb->vnn_map->generation != hdr->generation) {
 257                         DEBUG(DEBUG_DEBUG,(__location__ " ctdb operation %u"
 258                                 " request %u"
 259                                 " length %u from node %u to %u had an"
 260                                 " invalid generation id:%u while our"
 261                                 " generation id is:%u\n",
 262                                  hdr->operation, hdr->reqid,
 263                                  hdr->length,
 264                                  hdr->srcnode, hdr->destnode,
 265                                  hdr->generation, ctdb->vnn_map->generation));
 266                         goto done;
 267                 }
 268         }
 269
 270         switch (hdr->operation) {
 271         case CTDB_REQ_CALL:
 272                 CTDB_INCREMENT_STAT(ctdb, node.req_call);
 273                 ctdb_request_call(ctdb, hdr);
 274                 break;
 275
 276         case CTDB_REPLY_CALL:
 277                 CTDB_INCREMENT_STAT(ctdb, node.reply_call);
 278                 ctdb_reply_call(ctdb, hdr);
 279                 break;
 280
 281         case CTDB_REPLY_ERROR:
 282                 CTDB_INCREMENT_STAT(ctdb, node.reply_error);
 283                 ctdb_reply_error(ctdb, hdr);
 284                 break;
 285
 286         case CTDB_REQ_DMASTER:
 287                 CTDB_INCREMENT_STAT(ctdb, node.req_dmaster);
 288                 ctdb_request_dmaster(ctdb, hdr);
 289                 break;
 290
 291         case CTDB_REPLY_DMASTER:
 292                 CTDB_INCREMENT_STAT(ctdb, node.reply_dmaster);
 293                 ctdb_reply_dmaster(ctdb, hdr);
 294                 break;
 295
 296         case CTDB_REQ_MESSAGE:
 297                 CTDB_INCREMENT_STAT(ctdb, node.req_message);
 298                 ctdb_request_message(ctdb, hdr);
 299                 break;
 300
 301         case CTDB_REQ_CONTROL:
 302                 CTDB_INCREMENT_STAT(ctdb, node.req_control);
 303                 ctdb_request_control(ctdb, hdr);
 304                 break;
 305
 306         case CTDB_REPLY_CONTROL:
 307                 CTDB_INCREMENT_STAT(ctdb, node.reply_control);
 308                 ctdb_reply_control(ctdb, hdr);
 309                 break;
 310
 311         case CTDB_REQ_KEEPALIVE:
 312                 CTDB_INCREMENT_STAT(ctdb, keepalive_packets_recv);
 313                 ctdb_request_keepalive(ctdb, hdr);
 314                 break;
 315
 316         case CTDB_REQ_TUNNEL:
 317                 CTDB_INCREMENT_STAT(ctdb, node.req_tunnel);
 318                 ctdb_request_tunnel(ctdb, hdr);
 319                 break;
 320
 321         default:
 322                 DEBUG(DEBUG_CRIT,("%s: Packet with unknown operation %u\n",
 323                          __location__, hdr->operation));
 324                 break;
 325         }
 326
 327 done:
 328         talloc_free(tmp_ctx);
 329 }
 330
 331
 332 /*
 333   called by the transport layer when a node is dead
 334 */
 335 void ctdb_node_dead(struct ctdb_node *node)
 336 {
 337         if (node->ctdb->methods == NULL) {
 338                 DBG_ERR("Can not restart transport while shutting down\n");
 339                 return;
 340         }
 341         node->ctdb->methods->restart(node);
 342
 343         if (node->flags & NODE_FLAGS_DISCONNECTED) {
 344                 DEBUG(DEBUG_INFO,("%s: node %s is already marked disconnected: %u connected\n",
 345                          node->ctdb->name, node->name,
 346                          node->ctdb->num_connected));
 347                 return;
 348         }
 349         node->ctdb->num_connected--;
 350         node->flags |= NODE_FLAGS_DISCONNECTED | NODE_FLAGS_UNHEALTHY;
 351         node->rx_cnt = 0;
 352         node->dead_count = 0;
 353
 354         DEBUG(DEBUG_ERR,("%s: node %s is dead: %u connected\n",
 355                  node->ctdb->name, node->name, node->ctdb->num_connected));
 356         ctdb_daemon_cancel_controls(node->ctdb, node);
 357 }
 358
 359 /*
 360   called by the transport layer when a node is connected
 361 */
 362 void ctdb_node_connected(struct ctdb_node *node)
 363 {
 364         if (!(node->flags & NODE_FLAGS_DISCONNECTED)) {
 365                 DEBUG(DEBUG_INFO,("%s: node %s is already marked connected: %u connected\n",
 366                          node->ctdb->name, node->name,
 367                          node->ctdb->num_connected));
 368                 return;
 369         }
 370         node->ctdb->num_connected++;
 371         node->dead_count = 0;
 372         node->flags &= ~NODE_FLAGS_DISCONNECTED;
 373         DEBUG(DEBUG_ERR,
 374               ("%s: connected to %s - %u connected\n",
 375                node->ctdb->name, node->name, node->ctdb->num_connected));
 376 }
 377
 378 struct queue_next {
 379         struct ctdb_context *ctdb;
 380         struct ctdb_req_header *hdr;
 381 };
 382
 383
 384 /*
 385   triggered when a deferred packet is due
 386  */
 387 static void queue_next_trigger(struct tevent_context *ev,
 388                                struct tevent_timer *te,
 389                                struct timeval t, void *private_data)
 390 {
 391         struct queue_next *q = talloc_get_type(private_data, struct queue_next);
 392         ctdb_input_pkt(q->ctdb, q->hdr);
 393         talloc_free(q);
 394 }
 395
 396 /*
 397   defer a packet, so it is processed on the next event loop
 398   this is used for sending packets to ourselves
 399  */
 400 static void ctdb_defer_packet(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
 401 {
 402         struct queue_next *q;
 403         q = talloc(ctdb, struct queue_next);
 404         if (q == NULL) {
 405                 DEBUG(DEBUG_ERR,(__location__ " Failed to allocate deferred packet\n"));
 406                 return;
 407         }
 408         q->ctdb = ctdb;
 409         q->hdr = talloc_memdup(q, hdr, hdr->length);
 410         if (q->hdr == NULL) {
 411                 talloc_free(q);
 412                 DEBUG(DEBUG_ERR,("Error copying deferred packet to self\n"));
 413                 return;
 414         }
 415 #if 0
 416         /* use this to put packets directly into our recv function */
 417         ctdb_input_pkt(q->ctdb, q->hdr);
 418 #else
 419         tevent_add_timer(ctdb->ev, q, timeval_zero(), queue_next_trigger, q);
 420 #endif
 421 }
 422
 423
 424 /*
 425   broadcast a packet to all nodes
 426 */
 427 static void ctdb_broadcast_packet_all(struct ctdb_context *ctdb,
 428                                       struct ctdb_req_header *hdr)
 429 {
 430         unsigned int i;
 431         for (i=0; i < ctdb->num_nodes; i++) {
 432                 if (ctdb->nodes[i]->flags & NODE_FLAGS_DELETED) {
 433                         continue;
 434                 }
 435                 hdr->destnode = ctdb->nodes[i]->pnn;
 436                 ctdb_queue_packet(ctdb, hdr);
 437         }
 438 }
 439
 440 /*
 441   broadcast a packet to all active nodes
 442 */
 443 static void ctdb_broadcast_packet_active(struct ctdb_context *ctdb,
 444                                          struct ctdb_req_header *hdr)
 445 {
 446         unsigned int i;
 447         for (i = 0; i < ctdb->num_nodes; i++) {
 448                 if (ctdb->nodes[i]->flags & NODE_FLAGS_INACTIVE) {
 449                         continue;
 450                 }
 451
 452                 hdr->destnode = ctdb->nodes[i]->pnn;
 453                 ctdb_queue_packet(ctdb, hdr);
 454         }
 455 }
 456
 457 /*
 458   broadcast a packet to all connected nodes
 459 */
 460 static void ctdb_broadcast_packet_connected(struct ctdb_context *ctdb,
 461                                             struct ctdb_req_header *hdr)
 462 {
 463         unsigned int i;
 464         for (i=0; i < ctdb->num_nodes; i++) {
 465                 if (ctdb->nodes[i]->flags & NODE_FLAGS_DELETED) {
 466                         continue;
 467                 }
 468                 if (!(ctdb->nodes[i]->flags & NODE_FLAGS_DISCONNECTED)) {
 469                         hdr->destnode = ctdb->nodes[i]->pnn;
 470                         ctdb_queue_packet(ctdb, hdr);
 471                 }
 472         }
 473 }
 474
 475 /*
 476   queue a packet or die
 477 */
 478 void ctdb_queue_packet(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
 479 {
 480         struct ctdb_node *node;
 481
 482         switch (hdr->destnode) {
 483         case CTDB_BROADCAST_ALL:
 484                 ctdb_broadcast_packet_all(ctdb, hdr);
 485                 return;
 486         case CTDB_BROADCAST_ACTIVE:
 487                 ctdb_broadcast_packet_active(ctdb, hdr);
 488                 return;
 489         case CTDB_BROADCAST_CONNECTED:
 490                 ctdb_broadcast_packet_connected(ctdb, hdr);
 491                 return;
 492         }
 493
 494         CTDB_INCREMENT_STAT(ctdb, node_packets_sent);
 495
 496         if (!ctdb_validate_pnn(ctdb, hdr->destnode)) {
 497                 DEBUG(DEBUG_CRIT,(__location__ " can't send to node %u that does not exist\n",
 498                          hdr->destnode));
 499                 return;
 500         }
 501
 502         node = ctdb->nodes[hdr->destnode];
 503
 504         if (node->flags & NODE_FLAGS_DELETED) {
 505                 DEBUG(DEBUG_ERR, (__location__ " Can not queue packet to DELETED node %d\n", hdr->destnode));
 506                 return;
 507         }
 508
 509         if (node->pnn == ctdb->pnn) {
 510                 ctdb_defer_packet(ctdb, hdr);
 511                 return;
 512         }
 513
 514         if (ctdb->methods == NULL) {
 515                 DEBUG(DEBUG_ALERT, (__location__ " Can not queue packet. "
 516                                     "Transport is DOWN\n"));
 517                 return;
 518         }
 519
 520         node->tx_cnt++;
 521         if (ctdb->methods->queue_pkt(node, (uint8_t *)hdr, hdr->length) != 0) {
 522                 ctdb_fatal(ctdb, "Unable to queue packet\n");
 523         }
 524 }
 525
 526
 527
 528
 529 /*
 530   a valgrind hack to allow us to get opcode specific backtraces
 531   very ugly, and relies on no compiler optimisation!
 532 */
 533 void ctdb_queue_packet_opcode(struct ctdb_context *ctdb, struct ctdb_req_header *hdr, unsigned opcode)
 534 {
 535         switch (opcode) {
 536 #define DO_OP(x) case x: ctdb_queue_packet(ctdb, hdr); break
 537                 DO_OP(1);
 538                 DO_OP(2);
 539                 DO_OP(3);
 540                 DO_OP(4);
 541                 DO_OP(5);
 542                 DO_OP(6);
 543                 DO_OP(7);
 544                 DO_OP(8);
 545                 DO_OP(9);
 546                 DO_OP(10);
 547                 DO_OP(11);
 548                 DO_OP(12);
 549                 DO_OP(13);
 550                 DO_OP(14);
 551                 DO_OP(15);
 552                 DO_OP(16);
 553                 DO_OP(17);
 554                 DO_OP(18);
 555                 DO_OP(19);
 556                 DO_OP(20);
 557                 DO_OP(21);
 558                 DO_OP(22);
 559                 DO_OP(23);
 560                 DO_OP(24);
 561                 DO_OP(25);
 562                 DO_OP(26);
 563                 DO_OP(27);
 564                 DO_OP(28);
 565                 DO_OP(29);
 566                 DO_OP(30);
 567                 DO_OP(31);
 568                 DO_OP(32);
 569                 DO_OP(33);
 570                 DO_OP(34);
 571                 DO_OP(35);
 572                 DO_OP(36);
 573                 DO_OP(37);
 574                 DO_OP(38);
 575                 DO_OP(39);
 576                 DO_OP(40);
 577                 DO_OP(41);
 578                 DO_OP(42);
 579                 DO_OP(43);
 580                 DO_OP(44);
 581                 DO_OP(45);
 582                 DO_OP(46);
 583                 DO_OP(47);
 584                 DO_OP(48);
 585                 DO_OP(49);
 586                 DO_OP(50);
 587                 DO_OP(51);
 588                 DO_OP(52);
 589                 DO_OP(53);
 590                 DO_OP(54);
 591                 DO_OP(55);
 592                 DO_OP(56);
 593                 DO_OP(57);
 594                 DO_OP(58);
 595                 DO_OP(59);
 596                 DO_OP(60);
 597                 DO_OP(61);
 598                 DO_OP(62);
 599                 DO_OP(63);
 600                 DO_OP(64);
 601                 DO_OP(65);
 602                 DO_OP(66);
 603                 DO_OP(67);
 604                 DO_OP(68);
 605                 DO_OP(69);
 606                 DO_OP(70);
 607                 DO_OP(71);
 608                 DO_OP(72);
 609                 DO_OP(73);
 610                 DO_OP(74);
 611                 DO_OP(75);
 612                 DO_OP(76);
 613                 DO_OP(77);
 614                 DO_OP(78);
 615                 DO_OP(79);
 616                 DO_OP(80);
 617                 DO_OP(81);
 618                 DO_OP(82);
 619                 DO_OP(83);
 620                 DO_OP(84);
 621                 DO_OP(85);
 622                 DO_OP(86);
 623                 DO_OP(87);
 624                 DO_OP(88);
 625                 DO_OP(89);
 626                 DO_OP(90);
 627                 DO_OP(91);
 628                 DO_OP(92);
 629                 DO_OP(93);
 630                 DO_OP(94);
 631                 DO_OP(95);
 632                 DO_OP(96);
 633                 DO_OP(97);
 634                 DO_OP(98);
 635                 DO_OP(99);
 636                 DO_OP(100);
 637         default:
 638                 ctdb_queue_packet(ctdb, hdr);
 639                 break;
 640         }
 641 }