4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 * Copyright (c) 2016 by Delphix. All rights reserved.
27 #pragma ident "%Z%%M% %I% %E% SMI"
30 * vs_eng.c manages the vs_engines array of scan engine.
31 * Access to the array and other private data is protected by vs_eng_mutex.
32 * A caller can wait for an available engine connection on vs_eng_cv
36 #include <sys/types.h>
37 #include <sys/synch.h>
38 #include <sys/socket.h>
39 #include <sys/filio.h>
40 #include <sys/ioctl.h>
41 #include <sys/debug.h>
43 #include <netinet/in.h>
44 #include <netinet/tcp.h>
45 #include <arpa/inet.h>
60 /* max connections per scan engine */
61 #define VS_CXN_MAX VS_VAL_SE_MAXCONN_MAX
64 * vs_eng_state_t - connection state
66 * Each configured scan engine supports up to vse_cfg.vep_maxconn
67 * connections. These connections are represented by a vs_connection_t
68 * which defines the connection state, associated socket descriptor
69 * and how long the connection has been available. A connection
70 * that has been available but unused for vs_inactivity_timeout
71 * seconds will be closed by the housekeeper thread.
73 * When a scan engine is reconfigured to have less connections
74 * (or is disabled) any of the superflous connections which are in
75 * AVAILABLE state are closed (DISCONNECTED). Others are set to
76 * CLOSE_PENDING to be closed (DISCONNECTED) when the engine is
77 * released (when the current request completes).
79 * +---------------------+
80 * |---------->| VS_ENG_DISCONNECTED |<-----------------|
81 * | +---------------------+ |
85 * | shutdown +---------------------+ reconfig | shutdown
86 * |<----------| VS_ENG_RESERVED | -----------| |
87 * | +---------------------+ | |
89 * | | +----------------------+
90 * | | connect | VS_ENG_CLOSE_PENDING |
91 * | | +----------------------+
93 * | shutdown +---------------------+ |
94 * |<----------| VS_ENG_INUSE |------------|
95 * | +---------------------+ reconfig/error
97 * | | release | eng_get
100 * | shutdown +---------------------+
101 * |<----------| VS_ENG_AVAILABLE |
102 * +---------------------+
107 VS_ENG_DISCONNECTED
= 0,
114 typedef struct vs_connection
{
115 vs_eng_state_t vsc_state
;
117 struct timeval vsc_avail_time
;
120 typedef struct vs_engine
{
121 vs_props_se_t vse_cfg
; /* host, port, maxconn */
122 int vse_inuse
; /* # connections in use */
124 vs_connection_t vse_cxns
[VS_CXN_MAX
];
127 static vs_engine_t vs_engines
[VS_SE_MAX
];
129 static int vs_eng_next
; /* round-robin "finger" */
130 static int vs_eng_count
; /* how many configured engines */
131 static int vs_eng_total_maxcon
; /* total configured connections */
132 static int vs_eng_total_inuse
; /* total connections in use */
133 static int vs_eng_wait_count
; /* # threads waiting for connection */
135 static pthread_mutex_t vs_eng_mutex
= PTHREAD_MUTEX_INITIALIZER
;
136 static pthread_cond_t vs_eng_cv
;
137 int vs_inactivity_timeout
= 60; /* seconds */
138 int vs_reuse_connection
= 1;
140 time_t vs_eng_wait
= VS_ENG_WAIT_DFLT
;
142 /* local functions */
143 static int vs_eng_connect(char *, int);
144 static boolean_t
vs_eng_check_errors(void);
145 static int vs_eng_find_connection(int *, int *, boolean_t
);
146 static int vs_eng_find_next(boolean_t
);
147 static int vs_eng_compare(int, char *, int);
148 static void vs_eng_config_close(vs_engine_t
*, int);
149 static void *vs_eng_housekeeper(void *);
153 /* non-blocking connect */
154 static int nbio_connect(int, const struct sockaddr
*, int);
155 int vs_connect_timeout
= 5000; /* milliseconds */
167 (void) pthread_cond_init(&vs_eng_cv
, NULL
);
168 (void) pthread_mutex_lock(&vs_eng_mutex
);
170 (void) memset(vs_engines
, 0, sizeof (vs_engine_t
) * VS_SE_MAX
);
172 vs_eng_total_maxcon
= 0;
173 vs_eng_total_inuse
= 0;
177 (void) pthread_mutex_unlock(&vs_eng_mutex
);
179 (void) pthread_create(&tid
, NULL
, vs_eng_housekeeper
, NULL
);
186 * Configure scan engine connections.
188 * If a scan engine has been reconfigured (different host or port)
189 * the scan engine's error count is reset.
191 * If the host/port has changed, the engine has been disabled
192 * or less connections are configured now, connections need
193 * to be closed or placed in CLOSE_PENDING state (vs_eng_config_close)
195 * vs_icap_config is invoked to reset engine-specific data stored
200 vs_eng_config(vs_props_all_t
*config
)
206 (void) pthread_mutex_lock(&vs_eng_mutex
);
209 vs_eng_total_maxcon
= 0;
211 for (i
= 0; i
< VS_SE_MAX
; i
++) {
212 cfg
= &config
->va_se
[i
];
213 eng
= &vs_engines
[i
];
215 if (vs_eng_compare(i
, cfg
->vep_host
, cfg
->vep_port
) != 0) {
216 vs_eng_config_close(eng
, 0);
217 eng
->vse_error
= B_FALSE
;
220 if (cfg
->vep_enable
) {
221 if (cfg
->vep_maxconn
< eng
->vse_cfg
.vep_maxconn
)
222 vs_eng_config_close(eng
, cfg
->vep_maxconn
);
225 vs_eng_total_maxcon
+= cfg
->vep_maxconn
;
228 vs_eng_config_close(eng
, 0);
229 (void) memset(&eng
->vse_cfg
, 0, sizeof (vs_props_se_t
));
232 vs_icap_config(i
, eng
->vse_cfg
.vep_host
, eng
->vse_cfg
.vep_port
);
235 if ((vs_eng_total_maxcon
<= 0) || (vs_eng_count
== 0))
236 syslog(LOG_NOTICE
, "Scan Engine - no engines configured");
238 (void) pthread_mutex_unlock(&vs_eng_mutex
);
243 * vs_eng_config_close
245 * If the host/port has changed, the engine has been disabled
246 * or less connections are configured now, connections need
247 * to be closed or placed in CLOSE_PENDING state
250 vs_eng_config_close(vs_engine_t
*eng
, int start_idx
)
253 vs_connection_t
*cxn
;
255 for (i
= start_idx
; i
< eng
->vse_cfg
.vep_maxconn
; i
++) {
256 cxn
= &(eng
->vse_cxns
[i
]);
258 switch (cxn
->vsc_state
) {
259 case VS_ENG_RESERVED
:
261 cxn
->vsc_state
= VS_ENG_CLOSE_PENDING
;
263 case VS_ENG_AVAILABLE
:
264 (void) close(cxn
->vsc_sockfd
);
265 cxn
->vsc_sockfd
= -1;
266 cxn
->vsc_state
= VS_ENG_DISCONNECTED
;
268 case VS_ENG_CLOSE_PENDING
:
269 case VS_ENG_DISCONNECTED
:
282 (void) pthread_cond_destroy(&vs_eng_cv
);
289 * Wakeup every (vs_inactivity_timeout / 2) seconds and close
290 * any connections that are in AVAILABLE state but have not
291 * been used for vs_inactivity_timeout seconds.
295 vs_eng_housekeeper(void *arg
)
301 vs_connection_t
*cxn
;
304 (void) sleep(vs_inactivity_timeout
/ 2);
306 if (vscand_get_state() == VS_STATE_SHUTDOWN
)
309 (void) gettimeofday(&now
, NULL
);
310 expire
= now
.tv_sec
- vs_inactivity_timeout
;
312 (void) pthread_mutex_lock(&vs_eng_mutex
);
313 for (i
= 0; i
< VS_SE_MAX
; i
++) {
314 eng
= &(vs_engines
[i
]);
315 for (j
= 0; j
< eng
->vse_cfg
.vep_maxconn
; j
++) {
316 cxn
= &(eng
->vse_cxns
[j
]);
318 if ((cxn
->vsc_state
== VS_ENG_AVAILABLE
) &&
319 (cxn
->vsc_avail_time
.tv_sec
< expire
)) {
320 (void) close(cxn
->vsc_sockfd
);
321 cxn
->vsc_sockfd
= -1;
322 cxn
->vsc_state
= VS_ENG_DISCONNECTED
;
326 (void) pthread_mutex_unlock(&vs_eng_mutex
);
336 * If the engine identified in conn (host, port) matches the
337 * engine in vs_engines set or clear the error state of the
338 * engine and update the error statistics.
340 * If error == 0, clear the error state(B_FALSE), else set
341 * the error state (B_TRUE) and increment engine error stats
344 vs_eng_set_error(vs_eng_ctx_t
*eng_ctx
, int error
)
346 int eidx
= eng_ctx
->vse_eidx
;
347 int cidx
= eng_ctx
->vse_cidx
;
350 (void) pthread_mutex_lock(&vs_eng_mutex
);
352 eng
= &(vs_engines
[eidx
]);
354 if (vs_eng_compare(eidx
, eng_ctx
->vse_host
, eng_ctx
->vse_port
) == 0)
355 eng
->vse_error
= (error
== 0) ? B_FALSE
: B_TRUE
;
358 eng
->vse_cxns
[cidx
].vsc_state
= VS_ENG_CLOSE_PENDING
;
359 vs_stats_eng_err(eng_ctx
->vse_engid
);
362 (void) pthread_mutex_unlock(&vs_eng_mutex
);
368 * Get next available scan engine connection.
369 * If retry == B_TRUE look for a scan engine with no errors.
371 * Returns: 0 - success
375 vs_eng_get(vs_eng_ctx_t
*eng_ctx
, boolean_t retry
)
377 struct timespec tswait
;
378 int eidx
, cidx
, sockfd
;
380 vs_connection_t
*cxn
;
382 (void) pthread_mutex_lock(&vs_eng_mutex
);
385 * If no engines connections configured OR
386 * retry and only one engine configured, give up
388 if ((vs_eng_total_maxcon
<= 0) ||
389 ((retry
== B_TRUE
) && (vs_eng_count
<= 1))) {
390 (void) pthread_mutex_unlock(&vs_eng_mutex
);
394 tswait
.tv_sec
= vs_eng_wait
;
397 while ((vscand_get_state() != VS_STATE_SHUTDOWN
) &&
398 (vs_eng_find_connection(&eidx
, &cidx
, retry
) == -1)) {
399 /* If retry and all configured engines have errors, give up */
400 if (retry
&& vs_eng_check_errors() == B_TRUE
) {
401 (void) pthread_mutex_unlock(&vs_eng_mutex
);
405 /* wait for a connection to become available */
407 if (pthread_cond_reltimedwait_np(&vs_eng_cv
, &vs_eng_mutex
,
409 syslog(LOG_NOTICE
, "Scan Engine "
410 "- timeout waiting for available engine");
412 (void) pthread_mutex_unlock(&vs_eng_mutex
);
418 if (vscand_get_state() == VS_STATE_SHUTDOWN
) {
419 (void) pthread_mutex_unlock(&vs_eng_mutex
);
423 eng
= &(vs_engines
[eidx
]);
424 cxn
= &(eng
->vse_cxns
[cidx
]);
426 /* update in use counts */
428 vs_eng_total_inuse
++;
430 /* update round-robin index */
432 vs_eng_next
= (eidx
== VS_SE_MAX
) ? 0 : eidx
+ 1;
434 /* populate vs_eng_ctx_t */
435 eng_ctx
->vse_eidx
= eidx
;
436 eng_ctx
->vse_cidx
= cidx
;
437 (void) strlcpy(eng_ctx
->vse_engid
, eng
->vse_cfg
.vep_engid
,
438 sizeof (eng_ctx
->vse_engid
));
439 (void) strlcpy(eng_ctx
->vse_host
, eng
->vse_cfg
.vep_host
,
440 sizeof (eng_ctx
->vse_host
));
441 eng_ctx
->vse_port
= eng
->vse_cfg
.vep_port
;
442 eng_ctx
->vse_sockfd
= cxn
->vsc_sockfd
;
444 if (cxn
->vsc_state
== VS_ENG_INUSE
) {
445 (void) pthread_mutex_unlock(&vs_eng_mutex
);
449 /* state == VS_ENG_RESERVED, need to connect */
451 (void) pthread_mutex_unlock(&vs_eng_mutex
);
453 sockfd
= vs_eng_connect(eng_ctx
->vse_host
, eng_ctx
->vse_port
);
455 /* retry a failed connection once */
458 sockfd
= vs_eng_connect(eng_ctx
->vse_host
, eng_ctx
->vse_port
);
462 syslog(LOG_NOTICE
, "Scan Engine - connection error (%s:%d) %s",
463 eng_ctx
->vse_host
, eng_ctx
->vse_port
,
464 errno
? strerror(errno
) : "");
465 vs_eng_set_error(eng_ctx
, 1);
466 vs_eng_release(eng_ctx
);
470 (void) pthread_mutex_lock(&vs_eng_mutex
);
471 switch (cxn
->vsc_state
) {
472 case VS_ENG_DISCONNECTED
:
473 /* SHUTDOWN occured */
474 (void) pthread_mutex_unlock(&vs_eng_mutex
);
475 vs_eng_release(eng_ctx
);
477 case VS_ENG_RESERVED
:
478 cxn
->vsc_state
= VS_ENG_INUSE
;
480 case VS_ENG_CLOSE_PENDING
:
481 /* reconfigure occured. Connection will be closed after use */
484 case VS_ENG_AVAILABLE
:
490 cxn
->vsc_sockfd
= sockfd
;
491 eng_ctx
->vse_sockfd
= sockfd
;
493 (void) pthread_mutex_unlock(&vs_eng_mutex
);
499 * vs_eng_check_errors
501 * Check if all engines with maxconn > 0 are in error state
503 * Returns: B_TRUE - all (valid) engines are in error state
504 * B_FALSE - otherwise
507 vs_eng_check_errors()
511 for (i
= 0; i
< VS_SE_MAX
; i
++) {
512 if (vs_engines
[i
].vse_cfg
.vep_maxconn
> 0 &&
513 (vs_engines
[i
].vse_error
== B_FALSE
))
522 * vs_eng_find_connection
524 * Identify the next engine to be used (vs_eng_find_next()).
525 * Select the engine's first connection in AVAILABLE state.
526 * If no connection is in AVAILABLE state, select the first
527 * that is in DISCONNECTED state.
530 * -1 no engine connections available (eng_idx & cxn_idx undefined)
533 vs_eng_find_connection(int *eng_idx
, int *cxn_idx
, boolean_t retry
)
537 vs_connection_t
*cxn
;
539 /* identify engine */
540 if ((idx
= vs_eng_find_next(retry
)) == -1)
543 eng
= &(vs_engines
[idx
]);
546 /* identify connection */
548 for (i
= 0; i
< eng
->vse_cfg
.vep_maxconn
; i
++) {
549 cxn
= &(eng
->vse_cxns
[i
]);
550 if (cxn
->vsc_state
== VS_ENG_AVAILABLE
) {
552 cxn
->vsc_state
= VS_ENG_INUSE
;
557 (cxn
->vsc_state
== VS_ENG_DISCONNECTED
)) {
565 eng
->vse_cxns
[idx
].vsc_state
= VS_ENG_RESERVED
;
574 * Returns: -1 no engine connections available
575 * idx of engine to use
578 vs_eng_find_next(boolean_t retry
)
582 for (i
= vs_eng_next
; i
< VS_SE_MAX
; i
++) {
583 if (vs_engines
[i
].vse_inuse
<
584 vs_engines
[i
].vse_cfg
.vep_maxconn
) {
585 if (!retry
|| (vs_engines
[i
].vse_error
== B_FALSE
))
590 for (i
= 0; i
< vs_eng_next
; i
++) {
591 if (vs_engines
[i
].vse_inuse
<
592 vs_engines
[i
].vse_cfg
.vep_maxconn
) {
593 if (!retry
|| (vs_engines
[i
].vse_error
== B_FALSE
))
606 vs_eng_release(const vs_eng_ctx_t
*eng_ctx
)
608 int eidx
= eng_ctx
->vse_eidx
;
609 int cidx
= eng_ctx
->vse_cidx
;
610 vs_connection_t
*cxn
;
612 (void) pthread_mutex_lock(&vs_eng_mutex
);
613 cxn
= &(vs_engines
[eidx
].vse_cxns
[cidx
]);
615 switch (cxn
->vsc_state
) {
616 case VS_ENG_DISCONNECTED
:
618 case VS_ENG_RESERVED
:
619 cxn
->vsc_state
= VS_ENG_DISCONNECTED
;
622 if (vs_reuse_connection
) {
623 cxn
->vsc_state
= VS_ENG_AVAILABLE
;
624 (void) gettimeofday(&cxn
->vsc_avail_time
, NULL
);
627 /* LINTED E_CASE_FALL_THROUGH - close connection */
628 case VS_ENG_CLOSE_PENDING
:
629 (void) close(cxn
->vsc_sockfd
);
630 cxn
->vsc_sockfd
= -1;
631 cxn
->vsc_state
= VS_ENG_DISCONNECTED
;
633 case VS_ENG_AVAILABLE
:
639 /* decrement in use counts */
640 vs_engines
[eidx
].vse_inuse
--;
641 vs_eng_total_inuse
--;
643 /* wake up next thread waiting for a connection */
644 (void) pthread_cond_signal(&vs_eng_cv
);
646 (void) pthread_mutex_unlock(&vs_eng_mutex
);
651 * vs_eng_close_connections
653 * Set vs_eng_total_maxcon to 0 to ensure no new engine sessions
655 * Close all open connections to abort in-progress scans.
656 * Set connection state to DISCONNECTED.
659 vs_eng_close_connections(void)
662 vs_connection_t
*cxn
;
664 (void) pthread_mutex_lock(&vs_eng_mutex
);
665 vs_eng_total_maxcon
= 0;
667 for (i
= 0; i
< VS_SE_MAX
; i
++) {
668 for (j
= 0; j
< VS_CXN_MAX
; j
++) {
669 cxn
= &(vs_engines
[i
].vse_cxns
[j
]);
671 switch (cxn
->vsc_state
) {
673 case VS_ENG_AVAILABLE
:
674 case VS_ENG_CLOSE_PENDING
:
675 (void) close(cxn
->vsc_sockfd
);
676 cxn
->vsc_sockfd
= -1;
678 case VS_ENG_DISCONNECTED
:
679 case VS_ENG_RESERVED
:
685 cxn
->vsc_state
= VS_ENG_DISCONNECTED
;
688 (void) pthread_mutex_unlock(&vs_eng_mutex
);
694 * open socket connection to remote scan engine
696 * Returns: sockfd or -1 (error)
699 vs_eng_connect(char *host
, int port
)
701 int rc
, sockfd
, opt_nodelay
, opt_keepalive
, opt_reuseaddr
, err_num
;
702 struct sockaddr_in addr
;
705 if ((sockfd
= socket(AF_INET
, SOCK_STREAM
, 0)) == -1)
708 hp
= getipnodebyname(host
, AF_INET
, 0, &err_num
);
710 (void) close(sockfd
);
714 (void) memset(&addr
, 0, sizeof (addr
));
715 (void) memcpy(&addr
.sin_addr
, hp
->h_addr
, hp
->h_length
);
716 addr
.sin_port
= htons(port
);
717 addr
.sin_family
= hp
->h_addrtype
;
720 #ifdef FIONBIO /* Use non-blocking mode for connect. */
721 rc
= nbio_connect(sockfd
, (struct sockaddr
*)&addr
,
722 sizeof (struct sockaddr
));
724 rc
= connect(sockfd
, (struct sockaddr
*)&addr
,
725 sizeof (struct sockaddr
));
733 (setsockopt(sockfd
, IPPROTO_TCP
, TCP_NODELAY
,
734 &opt_nodelay
, sizeof (opt_nodelay
)) < 0) ||
735 (setsockopt(sockfd
, SOL_SOCKET
, SO_KEEPALIVE
,
736 &opt_keepalive
, sizeof (opt_keepalive
)) < 0) ||
737 (setsockopt(sockfd
, SOL_SOCKET
, SO_REUSEADDR
,
738 &opt_reuseaddr
, sizeof (opt_reuseaddr
)) < 0)) {
739 (void) close(sockfd
);
750 * Attempt to do a non-blocking connect call.
751 * Wait for a maximum of "vs_connect_timeout" millisec, then check for
752 * socket error to determine if connect successful or not.
756 nbio_connect(int sockfd
, const struct sockaddr
*sa
, int sa_len
)
760 int error
, len
= sizeof (error
);
763 if ((ioctl(sockfd
, FIONBIO
, &nbio
)) < 0)
764 return (connect(sockfd
, sa
, sa_len
));
766 if ((rc
= connect(sockfd
, sa
, sa_len
)) != 0) {
767 if (errno
== EINPROGRESS
|| errno
== EINTR
) {
770 pfd
.events
= POLLOUT
;
773 if ((rc
= poll(&pfd
, 1, vs_connect_timeout
)) <= 0) {
779 (POLLHUP
| POLLERR
| POLLNVAL
)) ||
780 (!(pfd
.revents
& POLLOUT
))) {
783 rc
= getsockopt(sockfd
, SOL_SOCKET
,
784 SO_ERROR
, &error
, &len
);
785 if (rc
!= 0 || error
!= 0)
795 (void) ioctl(sockfd
, FIONBIO
, &nbio
);
803 * vs_eng_scanstamp_current
805 * Check if scanstamp matches that of ANY engine with no errors.
806 * We cannot include engines with errors as they may have been
807 * inaccessible for a long time and thus we may have an old
808 * scanstamp value for them.
809 * If a match is found the scanstamp is considered to be current
811 * returns: 1 if current, 0 otherwise
814 vs_eng_scanstamp_current(vs_scanstamp_t scanstamp
)
818 /* if scan stamp is null, not current */
819 if (scanstamp
[0] == '\0')
822 /* if scanstamp matches that of any enabled engine with no errors */
823 (void) pthread_mutex_lock(&vs_eng_mutex
);
824 for (i
= 0; i
< VS_SE_MAX
; i
++) {
825 if ((vs_engines
[i
].vse_cfg
.vep_enable
) &&
826 (vs_engines
[i
].vse_error
== B_FALSE
) &&
827 (vs_icap_compare_scanstamp(i
, scanstamp
) == 0))
830 (void) pthread_mutex_unlock(&vs_eng_mutex
);
832 return ((i
< VS_SE_MAX
) ? 1 : 0);
838 * compare host and port with that stored for engine idx
840 * Returns: 0 - if equal
843 vs_eng_compare(int idx
, char *host
, int port
)
845 if (vs_engines
[idx
].vse_cfg
.vep_port
!= port
)
848 if (strcmp(vs_engines
[idx
].vse_cfg
.vep_host
, host
) != 0)