2 * Connection oriented routing user space utils
3 * Copyright (C) 2009-2020
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version 2
10 * of the License, or (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
26 struct route_list localneighs
;
27 struct service_list localservices
;
29 __u8 local_query_status
= QUERY_STATUS_NOTQUERIED
;
30 __u64 local_last_query_time
= 0;
32 struct list_head node_list_reachable
;
33 struct list_head node_list_unreachable
;
34 struct list_head node_list_recalcneeded
;
37 #warning todo refcnt of struct node
39 int service_list_contains(struct service_list
*services
, __be32 port
)
43 for (u
=0;u
<services
->numports
;u
++) {
44 if (services
->ports
[u
] == port
) {
52 struct node
* try_find_neigh_byaddr(__be64 addr
)
54 struct list_head
*curr
= node_list_reachable
.next
;
56 while (curr
!= &node_list_reachable
) {
57 struct node
*currnode
= container_of(curr
, struct node
,
60 if (currnode
->addr
== addr
)
69 struct node
* try_find_neigh_byservice(__be32 serviceport
)
71 struct list_head
*curr
= node_list_reachable
.next
;
73 while (curr
!= &node_list_reachable
) {
74 struct node
*currnode
= container_of(curr
, struct node
,
77 if (service_list_contains(&(currnode
->services
), serviceport
)) {
87 struct e2eroute
get_e2eroute_localhost(void)
90 bzero(&ret
, sizeof(struct e2eroute
));
95 struct e2eroute
get_e2eroute(struct node
*node
)
99 struct node
*currnode
;
104 bzero(&ret
, sizeof(struct e2eroute
));
106 if (get_static_route(node
->addr
, &ret
) == 0)
110 for (currnode
= node
;currnode
!= 0;currnode
= currnode
->prevhop
) {
112 ASSERT(currnode
->prevhop
!= 0 || currnode
->is_neighbor
== 1);
115 ret
.addr
= calloc(8, ret
.numhops
);
117 for (currnode
= node
;currnode
!= 0;currnode
= currnode
->prevhop
) {
119 ret
.addr
[u
-1] = node
->addr
;
122 ASSERT(currnode
->prevhop
!= 0 || currnode
->is_neighbor
== 1);
129 #warning todo call from rds_connect
130 void free_e2eroute_addr(struct e2eroute
*e2er
)
132 if (e2er
->numhops
== 0) {
133 ASSERT(e2er
->addr
== 0);
135 ASSERT(e2er
->addr
!= 0);
142 bzero(e2er
, sizeof(struct e2eroute
));
146 int connect_to_host_recv(int fd
, struct libcor_nonblock_resumeinfo
*lnr
,
147 struct nonblock_resumeinfo_connect_to_host_recv
*rcr
,
148 struct e2eroute e2er
)
150 if (rcr
->state
== 0) {
152 rcr
->numhops
= e2er
.numhops
;
158 ASSERT(rcr
->fd
== fd
);
159 ASSERT(rcr
->numhops
== e2er
.numhops
);
161 for (;rcr
->u
< rcr
->numhops
;rcr
->u
++) {
162 int rc
= read_resp_nonblock(fd
, 0, lnr
, 0, 0, 0);
166 bzero(rcr
, sizeof(struct nonblock_resumeinfo_connect_to_host_recv
));
170 int connect_to_host_send(int fd
,
171 struct libcor_nonblock_resumeinfo
*lnr
,
172 struct nonblock_resumeinfo_connect_to_host_send
*cth
,
173 struct e2eroute e2er
, int flush
)
175 if (cth
->state
== 0) {
184 ASSERT(cth
->fd
== fd
);
185 ASSERT(cth
->e2er
.numhops
== e2er
.numhops
);
187 for (;cth
->u
< e2er
.numhops
;) {
188 int flush1
= (flush
&& cth
->u
== e2er
.numhops
- 1);
189 int rc
= send_connect_neigh_nonblock(fd
, lnr
,
190 e2er
.addr
[cth
->u
], flush1
);
195 bzero(cth
, sizeof(struct nonblock_resumeinfo_connect_to_host_send
));
199 static void add_neigh(void *ptr
, __be64 addr
, __u32 latency_us
)
201 struct route_list
*list
= (struct route_list
*) ptr
;
205 printf("add neighbor %llx latency %u\n", ntohll(addr
), latency_us
);
210 if (list
->numroutes
>= list
->rows_alloc
)
213 ASSERT(list
->routes
[list
->numroutes
].dst
== 0);
215 node
= try_find_neigh_byaddr(addr
);
218 node
= calloc(1, sizeof(struct node
));
220 list_add_tail(&(node
->node_list
), &node_list_reachable
);
223 list
->routes
[list
->numroutes
].dst
= node
;
224 list
->routes
[list
->numroutes
].latency_us
= latency_us
;
228 static void init_neighlist(void *ptr
, __u32 numneighs
)
230 struct route_list
*list
= (struct route_list
*) ptr
;
233 printf("init neighlist %u\n", numneighs
);
235 if (list
->routes
!= 0) {
242 list
->rows_alloc
= (__u16
) numneighs
;
244 list
->routes
= calloc(numneighs
, sizeof(struct route
));
247 static void add_service(void *ptr
, __be32 port
)
249 struct service_list
*list
= (struct service_list
*) ptr
;
252 printf("add service %u\n", ntohs(port
));
254 if (list
->numports
>= list
->rows_alloc
)
257 list
->ports
[list
->numports
] = port
;
261 static void init_servicelist(void *ptr
, __u32 numservices
)
263 struct service_list
*list
= (struct service_list
*) ptr
;
266 printf("init servicelist %u\n", numservices
);
268 if (list
->ports
!= 0) {
272 if (numservices
> 16)
274 list
->rows_alloc
= (__u16
) numservices
;
276 list
->ports
= calloc(numservices
, 4);
280 static int _export_servicelist(int fd
, __be64 addr
, __u32 cost
,
281 struct service_list
*services
)
285 for (u
=0;u
<services
->numports
;u
++) {
286 int rc
= dprintf(fd
, "%llx:%u:%u\n", ntohll(addr
),
287 ntohl(services
->ports
[u
]), cost
);
297 #warning todo move to /run/
298 static void export_servicelist(void)
300 char *servicelist_txtfile
= "/var/lib/cor/services/txt";
301 char *servicelist_tmpfile
= "/var/lib/cor/services/tmp";
304 struct list_head
*curr
;
306 if (export_servicelist_enabled
== 0)
309 fd
= open(servicelist_tmpfile
, O_WRONLY
| O_CREAT
| O_TRUNC
, 0644);
311 perror("export servicelist");
315 if (_export_servicelist(fd
, 0, 0, &localservices
) != 0)
318 curr
= node_list_reachable
.next
;
319 while (curr
!= &node_list_reachable
) {
320 struct node
*currnode
= container_of(curr
, struct node
,
325 ASSERT(has_localaddr
!= 0 || localaddr
== 0);
326 if (currnode
->addr
== 0)
328 if (currnode
->addr
== localaddr
)
331 if (_export_servicelist(fd
, currnode
->addr
,
332 currnode
->metric
, &(currnode
->services
)) != 0)
339 if (rename(servicelist_tmpfile
, servicelist_txtfile
) != 0) {
340 perror("export servicelist");
348 unlink(servicelist_tmpfile
);
352 static void _recalc_routes_add_recalcneeded(struct node
*node
)
354 struct list_head
*curr
;
356 curr
= node_list_recalcneeded
.prev
;
357 while (curr
!= &node_list_recalcneeded
) {
358 struct node
*currnode
= container_of(curr
, struct node
,
361 if (currnode
->metric
> node
->metric
) {
362 list_add(&(node
->node_list
), &(currnode
->node_list
));
369 list_add(&(node
->node_list
), &node_list_recalcneeded
);
372 static void _recalc_routes(struct node
*node
)
374 struct route_list
*routes
= &(node
->routes
);
378 for (u
=0;u
<routes
->numroutes
;u
++) {
379 struct node
*target
= routes
->routes
[u
].dst
;
381 __u32 metric
= node
->metric
+
382 routes
->routes
[u
].latency_us
+
383 EXTRA_LATENCY_PERHOP_US
;
385 ASSERT(node
->metric
!= 0);
386 ASSERT(node
->is_neighbor
== 1 || node
->prevhop
!= 0);
388 if (metric
< node
->metric
)
391 if (target
->metric
<= metric
)
394 target
->metric
= metric
;
395 target
->prevhop
= node
;
397 list_del(&(target
->node_list
));
398 _recalc_routes_add_recalcneeded(target
);
401 list_del(&(node
->node_list
));
402 list_add_tail(&(node
->node_list
), &node_list_reachable
);
405 static void _recalc_routes_localneighs(void)
408 for (u
=0;u
<localneighs
.numroutes
;u
++) {
409 struct node
*target
= localneighs
.routes
[u
].dst
;
411 target
->is_neighbor
= 1;
412 target
->metric
= localneighs
.routes
[u
].latency_us
+
413 EXTRA_LATENCY_PERHOP_US
;
416 list_del(&(target
->node_list
));
417 _recalc_routes_add_recalcneeded(target
);
421 static void _recalc_routes_reset_reachable(void)
423 struct list_head
*curr
= node_list_reachable
.next
;
424 while (curr
!= &node_list_reachable
) {
425 struct node
*currnode
= container_of(curr
, struct node
,
428 currnode
->is_neighbor
= 0;
429 currnode
->prevhop
= 0;
430 currnode
->metric
= 0;
435 list_add(&node_list_unreachable
, &node_list_reachable
);
436 list_del(&node_list_reachable
);
437 init_list_head(&node_list_reachable
);
440 static void recalc_routes(void)
443 printf("recalc_routes\n");
445 _recalc_routes_reset_reachable();
447 _recalc_routes_localneighs();
449 while (!list_empty(&node_list_recalcneeded
)) {
450 struct node
*currnode
= container_of(
451 node_list_recalcneeded
.next
, struct node
,
454 _recalc_routes(currnode
);
457 /* we cannot free struct node here, because nonblock_resumeinfo may still have a pointer */
458 /*while (!list_empty(&node_list_unreachable)) {
459 struct node *currnode = container_of(
460 node_list_unreachable.next, struct node,
463 list_del(&(currnode->node_list));
465 if (currnode->addr != 0) {
466 free(currnode->addr);
473 export_servicelist();
477 static void discover_localhost()
482 __u8 bindata_noresp
= 0;
484 local_last_query_time
= get_time_nsec();
486 fd
= socket(PF_COR
, SOCK_RAW
, PROTO_COR_RAW
);
492 if (setsockopts_tos(fd
, COR_TOS_HIGH_LATENCY
) != 0) {
493 perror("setsockopt");
497 if (connect(fd
, 0, 0) != 0) {
503 rc
= send_list_neigh(fd
, 1);
507 rc
= read_resp(fd
, 1, &bindata_noresp
);
511 if (bindata_noresp
!= 0) {
512 init_neighlist(&localneighs
, 0);
515 __u32 resp_bin_len
= 0;
517 rc
= read_resp_bin(fd
, &resp_bin
, &resp_bin_len
);
521 rc
= parse_neigh_list(resp_bin
, resp_bin_len
, DEFAULT_LATENCY_US
,
522 &localneighs
, init_neighlist
, add_neigh
);
531 rc
= send_list_services(fd
, 1);
535 rc
= read_resp(fd
, 1, &bindata_noresp
);
539 if (bindata_noresp
!= 0) {
540 init_servicelist(&localservices
, 0);
543 __u32 resp_bin_len
= 0;
545 rc
= read_resp_bin(fd
, &resp_bin
, &resp_bin_len
);
549 rc
= parse_service_list(resp_bin
, resp_bin_len
, &localservices
,
550 init_servicelist
, add_service
);
560 local_query_status
= QUERY_STATUS_QUERIED
;
568 local_query_status
= QUERY_STATUS_ERROR
;
572 void _discover_network(int fd
, struct nonblock_resumeinfo
*nr
)
575 struct libcor_nonblock_resumeinfo
*lnr
;
576 int rc
= RC_CONNBROKEN
;
578 __u8 bindata_noresp
= 0;
580 lnr
= &(nr
->data
.discover_network
.lnr
);
582 ASSERT(nr
->type
== EPOLLDATA_DISCOVERNETWORK
);
584 node
= nr
->data
.discover_network
.node
;
586 if (nr
->data
.discover_network
.state
== 1) {
588 } else if (nr
->data
.discover_network
.state
== 2) {
590 } else if (nr
->data
.discover_network
.state
== 3) {
592 } else if (nr
->data
.discover_network
.state
== 4) {
594 } else if (nr
->data
.discover_network
.state
== 5) {
596 } else if (nr
->data
.discover_network
.state
== 6) {
598 } else if (nr
->data
.discover_network
.state
== 7) {
600 } else if (nr
->data
.discover_network
.state
!= 0) {
605 printf("discover resume neighbor %llx\n", ntohll(node
->addr
));
607 nr
->data
.discover_network
.e2er
= get_e2eroute(node
);
609 nr
->data
.discover_network
.state
= 1;
611 rc
= connect_to_host_send(fd
, lnr
,
612 &(nr
->data
.discover_network
.connect_send
),
613 nr
->data
.discover_network
.e2er
, 0);
615 if (check_rc(rc
, "load_neigh_list: connect_to_host_send error"))
618 rc
= send_list_neigh_nonblock(fd
, lnr
, 0);
619 nr
->data
.discover_network
.state
= 2;
620 if (check_rc(rc
, "load_neigh_list: send_list_neigh error"))
624 rc
= send_list_services_nonblock(fd
, lnr
, 1);
625 nr
->data
.discover_network
.state
= 3;
626 if (check_rc(rc
, "load_neigh_list: send_list_services error"))
630 rc
= connect_to_host_recv(fd
, lnr
,
631 &(nr
->data
.discover_network
.connect_recv
),
632 nr
->data
.discover_network
.e2er
);
633 if (check_rc(rc
, "load_neigh_list: connect_to_host_recv error"))
636 free_e2eroute_addr(&nr
->data
.discover_network
.e2er
);
638 nr
->data
.discover_network
.state
= 4;
640 rc
= read_resp_nonblock(fd
, 1, lnr
, 0, 0, &bindata_noresp
);
641 if (check_rc(rc
, "load_neigh_list: read_resp error"))
644 if (bindata_noresp
!= 0) {
645 init_neighlist(&(node
->routes
), 0);
648 __u32 resp_bin_len
= 0;
650 nr
->data
.discover_network
.state
= 5;
652 rc
= read_resp_bin_nonblock(fd
, lnr
, &resp_bin
, &resp_bin_len
);
653 if (check_rc(rc
, "load_neigh_list: read_resp_bin_nonblock (neighborlest) error"))
656 rc
= parse_neigh_list(resp_bin
, resp_bin_len
, DEFAULT_LATENCY_US
,
657 &(node
->routes
), init_neighlist
, add_neigh
);
663 ASSERT(rc
!= RC_WOULDBLOCK
);
664 if (check_rc(rc
, "load_neigh_list: parse_neigh_list error"))
668 nr
->data
.discover_network
.state
= 6;
670 rc
= read_resp_nonblock(fd
, 1, lnr
, 0, 0, &bindata_noresp
);
671 if (check_rc(rc
, "load_neigh_list: read_resp error"))
674 if (bindata_noresp
!= 0) {
676 __u32 resp_bin_len
= 0;
678 nr
->data
.discover_network
.state
= 7;
680 rc
= read_resp_bin_nonblock(fd
, lnr
, &resp_bin
, &resp_bin_len
);
681 if (check_rc(rc
, "load_neigh_list: read_resp_bin_nonblock (servicelist) error"))
684 rc
= parse_service_list(resp_bin
, resp_bin_len
, &(node
->services
),
685 init_servicelist
, add_service
);
691 ASSERT(rc
!= RC_WOULDBLOCK
);
692 if (check_rc(rc
, "load_neigh_list: parse_service_list error"))
696 node
->query_status
= QUERY_STATUS_QUERIED
;
699 //printf("load_neigh_list state %d\n", nr->data.discover_network.state);
700 if (rc
!= RC_WOULDBLOCK
) {
701 free_e2eroute_addr(&nr
->data
.discover_network
.e2er
);
703 //printf("load_neigh_list rc %d\n", rc);
708 if (rc
== RC_CONNBROKEN
)
709 node
->query_status
= QUERY_STATUS_ERROR
;
710 node
->query_in_progress
= 0;
714 static int _discover_network_start_load_node(struct node
*node
)
718 struct nonblock_resumeinfo
*nr
;
720 struct epoll_event epe
;
723 printf("discover start neighbor %llx\n", ntohll(node
->addr
));
725 ASSERT(node
->query_in_progress
== 0);
726 node
->query_in_progress
= 1;
727 node
->last_query_time
= get_time_nsec();
729 fd
= socket(PF_COR
, SOCK_RAW
, PROTO_COR_RAW
);
735 if (setsockopts_tos(fd
, COR_TOS_HIGH_LATENCY
) != 0) {
737 perror("setsockopt");
741 if (connect(fd
, 0, 0) != 0) {
749 nr
= (struct nonblock_resumeinfo
*)
750 malloc(sizeof(struct nonblock_resumeinfo
));
751 bzero(nr
, sizeof(struct nonblock_resumeinfo
));
753 nr
->type
= EPOLLDATA_DISCOVERNETWORK
;
754 nr
->data
.discover_network
.node
= node
;
756 bzero(&epe
, sizeof(struct epoll_event
));
757 epe
.events
= (EPOLLIN
| EPOLLOUT
| EPOLLRDHUP
| EPOLLERR
| EPOLLET
);
759 epoll_ctl(epoll_fd
, EPOLL_CTL_ADD
, fd
, &epe
);
763 node
->query_status
= QUERY_STATUS_ERROR
;
764 node
->query_in_progress
= 0;
768 void discover_network(void)
770 struct list_head
*curr
;
772 __u64 time_nsec
= get_time_nsec();
774 if (local_query_status
!= QUERY_STATUS_QUERIED
||
775 local_last_query_time
+ 1000000L < time_nsec
) {
777 printf("discover localhost\n");
778 discover_localhost();
781 curr
= node_list_reachable
.next
;
783 while (curr
!= &node_list_reachable
) {
784 struct node
*currnode
= container_of(curr
, struct node
,
787 if (currnode
->query_in_progress
== 0 && (
788 currnode
->query_status
==
789 QUERY_STATUS_NOTQUERIED
||
790 currnode
->last_query_time
+ 30000000L <
792 _discover_network_start_load_node(currnode
);
799 int initial_discovery_inprogress(void)
801 struct list_head
*curr
= node_list_reachable
.next
;
803 while (curr
!= &node_list_reachable
) {
804 struct node
*currnode
= container_of(curr
, struct node
,
807 if (currnode
->query_status
== QUERY_STATUS_NOTQUERIED
&&
808 currnode
->query_in_progress
!= 0)
817 void init_nodedb(void)
819 init_list_head(&node_list_reachable
);
820 init_list_head(&node_list_unreachable
);
821 init_list_head(&node_list_recalcneeded
);