7 /* dictionary manager interface to MySQL databases
9 /* #include <dict_mysql.h>
11 /* DICT *dict_mysql_open(name, open_flags, dict_flags)
16 /* dict_mysql_open() creates a dictionary of type 'mysql'. This
17 /* dictionary is an interface for the postfix key->value mappings
18 /* to mysql. The result is a pointer to the installed dictionary,
19 /* or a null pointer in case of problems.
21 /* The mysql dictionary can manage multiple connections to different
22 /* sql servers on different hosts. It assumes that the underlying data
23 /* on each host is identical (mirrored) and maintains one connection
24 /* at any given time. If any connection fails, any other available
25 /* ones will be opened and used. The intent of this feature is to eliminate
26 /* a single point of failure for mail systems that would otherwise rely
27 /* on a single mysql server.
31 /* Either the path to the MySQL configuration file (if it starts
32 /* with '/' or '.'), or the prefix which will be used to obtain
33 /* main.cf configuration parameters for this search.
35 /* In the first case, the configuration parameters below are
36 /* specified in the file as \fIname\fR=\fBvalue\fR pairs.
38 /* In the second case, the configuration parameters are
39 /* prefixed with the value of \fIname\fR and an underscore,
40 /* and they are specified in main.cf. For example, if this
41 /* value is \fImysqlsource\fR, the parameters would look like
42 /* \fImysqlsource_user\fR, \fImysqlsource_table\fR, and so on.
45 /* reference for outside use.
51 /* Configuration parameters:
53 /* The parameters encodes a number of pieces of information:
54 /* username, password, databasename, table, select_field,
55 /* where_field, and hosts:
57 /* Username for connecting to the database.
59 /* Password for the above.
61 /* Name of the database.
63 /* List of domains the queries should be restricted to. If
64 /* specified, only FQDN addresses whose domain parts matching this
65 /* list will be queried against the SQL database. Lookups for
66 /* partial addresses are also supressed. This can significantly
67 /* reduce the query load on the server.
69 /* Query template, before the query is actually issued, variable
70 /* substitutions are performed. See mysql_table(5) for details. If
71 /* No query is specified, the legacy variables \fItable\fR,
72 /* \fIselect_field\fR, \fIwhere_field\fR and \fIadditional_conditions\fR
73 /* are used to construct the query template.
74 /* .IP \fIresult_format\fR
75 /* The format used to expand results from queries. Substitutions
76 /* are performed as described in mysql_table(5). Defaults to returning
77 /* the lookup result unchanged.
78 /* .IP expansion_limit
79 /* Limit (if any) on the total number of lookup result values. Lookups which
80 /* exceed the limit fail with dict_errno=DICT_ERR_RETRY. Note that each
81 /* non-empty (and non-NULL) column of a multi-column result row counts as
84 /* When \fIquery\fR is not set, name of the table used to construct the
85 /* query string. This provides compatibility with older releases.
86 /* .IP \fIselect_field\fR
87 /* When \fIquery\fR is not set, name of the result field used to
88 /* construct the query string. This provides compatibility with older
90 /* .IP \fIwhere_field\fR
91 /* When \fIquery\fR is not set, name of the where clause field used to
92 /* construct the query string. This provides compatibility with older
94 /* .IP \fIadditional_conditions\fR
95 /* When \fIquery\fR is not set, additional where clause conditions used
96 /* to construct the query string. This provides compatibility with older
99 /* List of hosts to connect to.
101 /* For example, if you want the map to reference databases of
102 /* the name "your_db" and execute a query like this: select
103 /* forw_addr from aliases where alias like '<some username>'
104 /* against any database called "vmailer_info" located on hosts
105 /* host1.some.domain and host2.some.domain, logging in as user
106 /* "vmailer" and password "passwd" then the configuration file
109 /* \fIuser\fR = \fBvmailer\fR
111 /* \fIpassword\fR = \fBpasswd\fR
113 /* \fIdbname\fR = \fBvmailer_info\fR
115 /* \fItable\fR = \fBaliases\fR
117 /* \fIselect_field\fR = \fBforw_addr\fR
119 /* \fIwhere_field\fR = \fBalias\fR
121 /* \fIhosts\fR = \fBhost1.some.domain\fR \fBhost2.some.domain\fR
122 /* .IP \fIadditional_conditions\fR
123 /* Backward compatibility when \fIquery\fR is not set, additional
124 /* conditions to the WHERE clause.
126 /* List of hosts to connect to.
128 /* For example, if you want the map to reference databases of
129 /* the name "your_db" and execute a query like this: select
130 /* forw_addr from aliases where alias like '<some username>'
131 /* against any database called "vmailer_info" located on hosts
132 /* host1.some.domain and host2.some.domain, logging in as user
133 /* "vmailer" and password "passwd" then the configuration file
136 /* \fIuser\fR = \fBvmailer\fR
138 /* \fIpassword\fR = \fBpasswd\fR
140 /* \fIdbname\fR = \fBvmailer_info\fR
142 /* \fItable\fR = \fBaliases\fR
144 /* \fIselect_field\fR = \fBforw_addr\fR
146 /* \fIwhere_field\fR = \fBalias\fR
148 /* \fIhosts\fR = \fBhost1.some.domain\fR \fBhost2.some.domain\fR
151 /* dict(3) generic dictionary manager
162 /* System library. */
163 #include "sys_defs.h"
166 #include <sys/socket.h>
167 #include <netinet/in.h>
168 #include <arpa/inet.h>
177 #ifdef STRCASECMP_IN_STRINGS_H
181 /* Utility library. */
185 #include "mymalloc.h"
188 #include "split_at.h"
189 #include "find_inet.h"
192 #include "stringops.h"
194 /* Global library. */
196 #include "cfg_parser.h"
197 #include "db_common.h"
199 /* Application-specific. */
201 #include "dict_mysql.h"
203 /* need some structs to help organize things */
209 unsigned type
; /* TYPEUNIX | TYPEINET */
210 unsigned stat
; /* STATUNTRIED | STATFAIL | STATCUR */
211 time_t ts
; /* used for attempting reconnection
212 * every so often if a host is down */
216 int len_hosts
; /* number of hosts */
217 HOST
**db_hosts
; /* the hosts on which the databases
233 #if defined(MYSQL_VERSION_ID) && MYSQL_VERSION_ID >= 40000
238 #define STATACTIVE (1<<0)
239 #define STATFAIL (1<<1)
240 #define STATUNTRIED (1<<2)
242 #define TYPEUNIX (1<<0)
243 #define TYPEINET (1<<1)
245 #define RETRY_CONN_MAX 100
246 #define RETRY_CONN_INTV 60 /* 1 minute */
247 #define IDLE_CONN_INTV 60 /* 1 minute */
249 /* internal function declarations */
250 static PLMYSQL
*plmysql_init(ARGV
*);
251 static MYSQL_RES
*plmysql_query(DICT_MYSQL
*, const char *, VSTRING
*, char *,
253 static void plmysql_dealloc(PLMYSQL
*);
254 static void plmysql_close_host(HOST
*);
255 static void plmysql_down_host(HOST
*);
256 static void plmysql_connect_single(HOST
*, char *, char *, char *);
257 static const char *dict_mysql_lookup(DICT
*, const char *);
258 DICT
*dict_mysql_open(const char *, int, int);
259 static void dict_mysql_close(DICT
*);
260 static void mysql_parse_config(DICT_MYSQL
*, const char *);
261 static HOST
*host_init(const char *);
263 /* dict_mysql_quote - escape SQL metacharacters in input string */
265 static void dict_mysql_quote(DICT
*dict
, const char *name
, VSTRING
*result
)
267 DICT_MYSQL
*dict_mysql
= (DICT_MYSQL
*) dict
;
268 int len
= strlen(name
);
269 int buflen
= 2*len
+ 1;
272 * We won't get integer overflows in 2*len + 1, because Postfix
273 * input keys have reasonable size limits, better safe than sorry.
276 msg_panic("dict_mysql_quote: integer overflow in 2*%d+1", len
);
277 VSTRING_SPACE(result
, buflen
);
279 #if defined(MYSQL_VERSION_ID) && MYSQL_VERSION_ID >= 40000
280 if (dict_mysql
->active_host
)
281 mysql_real_escape_string(dict_mysql
->active_host
->db
,
282 vstring_end(result
), name
, len
);
285 mysql_escape_string(vstring_end(result
), name
, len
);
287 VSTRING_SKIP(result
);
290 /* dict_mysql_lookup - find database entry */
292 static const char *dict_mysql_lookup(DICT
*dict
, const char *name
)
294 const char *myname
= "dict_mysql_lookup";
295 DICT_MYSQL
*dict_mysql
= (DICT_MYSQL
*)dict
;
296 PLMYSQL
*pldb
= dict_mysql
->pldb
;
297 MYSQL_RES
*query_res
;
299 static VSTRING
*result
;
300 static VSTRING
*query
;
306 db_quote_callback_t quote_func
= dict_mysql_quote
;
311 * Optionally fold the key.
313 if (dict
->flags
& DICT_FLAG_FOLD_FIX
) {
314 if (dict
->fold_buf
== 0)
315 dict
->fold_buf
= vstring_alloc(10);
316 vstring_strcpy(dict
->fold_buf
, name
);
317 name
= lowercase(vstring_str(dict
->fold_buf
));
321 * If there is a domain list for this map, then only search for
322 * addresses in domains on the list. This can significantly reduce
323 * the load on the server.
325 if (db_common_check_domain(dict_mysql
->ctx
, name
) == 0) {
327 msg_info("%s: Skipping lookup of '%s'", myname
, name
);
331 #define INIT_VSTR(buf, len) do { \
333 buf = vstring_alloc(len); \
334 VSTRING_RESET(buf); \
335 VSTRING_TERMINATE(buf); \
338 INIT_VSTR(query
, 10);
341 * Suppress the lookup if the query expansion is empty
343 * This initial expansion is outside the context of any
344 * specific host connection, we just want to check the
345 * key pre-requisites, so when quoting happens separately
346 * for each connection, we don't bother with quoting...
348 #if defined(MYSQL_VERSION_ID) && MYSQL_VERSION_ID >= 40000
351 if (!db_common_expand(dict_mysql
->ctx
, dict_mysql
->query
,
352 name
, 0, query
, quote_func
))
355 /* do the query - set dict_errno & cleanup if there's an error */
356 if ((query_res
= plmysql_query(dict_mysql
, name
, query
,
358 dict_mysql
->username
,
359 dict_mysql
->password
)) == 0) {
360 dict_errno
= DICT_ERR_RETRY
;
364 numrows
= mysql_num_rows(query_res
);
366 msg_info("%s: retrieved %d rows", myname
, numrows
);
368 mysql_free_result(query_res
);
372 INIT_VSTR(result
, 10);
374 for (expansion
= i
= 0; i
< numrows
&& dict_errno
== 0; i
++) {
375 row
= mysql_fetch_row(query_res
);
376 for (j
= 0; j
< mysql_num_fields(query_res
); j
++) {
377 if (db_common_expand(dict_mysql
->ctx
, dict_mysql
->result_format
,
378 row
[j
], name
, result
, 0)
379 && dict_mysql
->expansion_limit
> 0
380 && ++expansion
> dict_mysql
->expansion_limit
) {
381 msg_warn("%s: %s: Expansion limit exceeded for key: '%s'",
382 myname
, dict_mysql
->parser
->name
, name
);
383 dict_errno
= DICT_ERR_RETRY
;
388 mysql_free_result(query_res
);
389 r
= vstring_str(result
);
390 return ((dict_errno
== 0 && *r
) ? r
: 0);
393 /* dict_mysql_check_stat - check the status of a host */
395 static int dict_mysql_check_stat(HOST
*host
, unsigned stat
, unsigned type
,
398 if ((host
->stat
& stat
) && (!type
|| host
->type
& type
)) {
399 /* try not to hammer the dead hosts too often */
400 if (host
->stat
== STATFAIL
&& host
->ts
> 0 && host
->ts
>= t
)
407 /* dict_mysql_find_host - find a host with the given status */
409 static HOST
*dict_mysql_find_host(PLMYSQL
*PLDB
, unsigned stat
, unsigned type
)
416 t
= time((time_t *) 0);
417 for (i
= 0; i
< PLDB
->len_hosts
; i
++) {
418 if (dict_mysql_check_stat(PLDB
->db_hosts
[i
], stat
, type
, t
))
424 1 + count
* (double) myrand() / (1.0 + RAND_MAX
) : 1;
426 for (i
= 0; i
< PLDB
->len_hosts
; i
++) {
427 if (dict_mysql_check_stat(PLDB
->db_hosts
[i
], stat
, type
, t
) &&
429 return PLDB
->db_hosts
[i
];
435 /* dict_mysql_get_active - get an active connection */
437 static HOST
*dict_mysql_get_active(PLMYSQL
*PLDB
, char *dbname
,
438 char *username
, char *password
)
440 const char *myname
= "dict_mysql_get_active";
442 int count
= RETRY_CONN_MAX
;
444 /* Try the active connections first; prefer the ones to UNIX sockets. */
445 if ((host
= dict_mysql_find_host(PLDB
, STATACTIVE
, TYPEUNIX
)) != NULL
||
446 (host
= dict_mysql_find_host(PLDB
, STATACTIVE
, TYPEINET
)) != NULL
) {
448 msg_info("%s: found active connection to host %s", myname
,
454 * Try the remaining hosts.
455 * "count" is a safety net, in case the loop takes more than
456 * RETRY_CONN_INTV and the dead hosts are no longer skipped.
458 while (--count
> 0 &&
459 ((host
= dict_mysql_find_host(PLDB
, STATUNTRIED
| STATFAIL
,
460 TYPEUNIX
)) != NULL
||
461 (host
= dict_mysql_find_host(PLDB
, STATUNTRIED
| STATFAIL
,
462 TYPEINET
)) != NULL
)) {
464 msg_info("%s: attempting to connect to host %s", myname
,
466 plmysql_connect_single(host
, dbname
, username
, password
);
467 if (host
->stat
== STATACTIVE
)
475 /* dict_mysql_event - callback: close idle connections */
477 static void dict_mysql_event(int unused_event
, char *context
)
479 HOST
*host
= (HOST
*) context
;
482 plmysql_close_host(host
);
486 * plmysql_query - process a MySQL query. Return MYSQL_RES* on success.
487 * On failure, log failure and try other db instances.
488 * on failure of all db instances, return 0;
489 * close unnecessary active connections
492 static MYSQL_RES
*plmysql_query(DICT_MYSQL
*dict_mysql
,
499 PLMYSQL
*PLDB
= dict_mysql
->pldb
;
503 while ((host
= dict_mysql_get_active(PLDB
, dbname
, username
, password
)) != NULL
) {
505 #if defined(MYSQL_VERSION_ID) && MYSQL_VERSION_ID >= 40000
507 * The active host is used to escape strings in the
508 * context of the active connection's character encoding.
510 dict_mysql
->active_host
= host
;
511 VSTRING_RESET(query
);
512 VSTRING_TERMINATE(query
);
513 db_common_expand(dict_mysql
->ctx
, dict_mysql
->query
,
514 name
, 0, query
, dict_mysql_quote
);
515 dict_mysql
->active_host
= 0;
518 if (!(mysql_query(host
->db
, vstring_str(query
)))) {
519 if ((res
= mysql_store_result(host
->db
)) == 0) {
520 msg_warn("mysql query failed: %s", mysql_error(host
->db
));
521 plmysql_down_host(host
);
524 msg_info("dict_mysql: successful query from host %s", host
->hostname
);
525 event_request_timer(dict_mysql_event
, (char *) host
, IDLE_CONN_INTV
);
529 msg_warn("mysql query failed: %s", mysql_error(host
->db
));
530 plmysql_down_host(host
);
538 * plmysql_connect_single -
539 * used to reconnect to a single database when one is down or none is
540 * connected yet. Log all errors and set the stat field of host accordingly
542 static void plmysql_connect_single(HOST
*host
, char *dbname
, char *username
, char *password
)
544 if ((host
->db
= mysql_init(NULL
)) == NULL
)
545 msg_fatal("dict_mysql: insufficient memory");
546 if (mysql_real_connect(host
->db
,
547 (host
->type
== TYPEINET
? host
->name
: 0),
552 (host
->type
== TYPEUNIX
? host
->name
: 0),
555 msg_info("dict_mysql: successful connection to host %s",
557 host
->stat
= STATACTIVE
;
559 msg_warn("connect to mysql server %s: %s",
560 host
->hostname
, mysql_error(host
->db
));
561 plmysql_down_host(host
);
565 /* plmysql_close_host - close an established MySQL connection */
566 static void plmysql_close_host(HOST
*host
)
568 mysql_close(host
->db
);
570 host
->stat
= STATUNTRIED
;
574 * plmysql_down_host - close a failed connection AND set a "stay away from
577 static void plmysql_down_host(HOST
*host
)
579 mysql_close(host
->db
);
581 host
->ts
= time((time_t *) 0) + RETRY_CONN_INTV
;
582 host
->stat
= STATFAIL
;
583 event_cancel_timer(dict_mysql_event
, (char *) host
);
586 /* mysql_parse_config - parse mysql configuration file */
588 static void mysql_parse_config(DICT_MYSQL
*dict_mysql
, const char *mysqlcf
)
590 const char *myname
= "mysqlname_parse";
596 p
= dict_mysql
->parser
= cfg_parser_alloc(mysqlcf
);
597 dict_mysql
->username
= cfg_get_str(p
, "user", "", 0, 0);
598 dict_mysql
->password
= cfg_get_str(p
, "password", "", 0, 0);
599 dict_mysql
->dbname
= cfg_get_str(p
, "dbname", "", 1, 0);
600 dict_mysql
->result_format
= cfg_get_str(p
, "result_format", "%s", 1, 0);
602 * XXX: The default should be non-zero for safety, but that is not
603 * backwards compatible.
605 dict_mysql
->expansion_limit
= cfg_get_int(dict_mysql
->parser
,
606 "expansion_limit", 0, 0, 0);
608 if ((dict_mysql
->query
= cfg_get_str(p
, "query", NULL
, 0, 0)) == 0) {
610 * No query specified -- fallback to building it from components
611 * (old style "select %s from %s where %s")
613 buf
= vstring_alloc(64);
614 db_common_sql_build_query(buf
, p
);
615 dict_mysql
->query
= vstring_export(buf
);
619 * Must parse all templates before we can use db_common_expand()
622 (void) db_common_parse(&dict_mysql
->dict
, &dict_mysql
->ctx
,
623 dict_mysql
->query
, 1);
624 (void) db_common_parse(0, &dict_mysql
->ctx
, dict_mysql
->result_format
, 0);
625 db_common_parse_domain(p
, dict_mysql
->ctx
);
628 * Maps that use substring keys should only be used with the full
631 if (db_common_dict_partial(dict_mysql
->ctx
))
632 dict_mysql
->dict
.flags
|= DICT_FLAG_PATTERN
;
634 dict_mysql
->dict
.flags
|= DICT_FLAG_FIXED
;
635 if (dict_mysql
->dict
.flags
& DICT_FLAG_FOLD_FIX
)
636 dict_mysql
->dict
.fold_buf
= vstring_alloc(10);
638 hosts
= cfg_get_str(p
, "hosts", "", 0, 0);
640 dict_mysql
->hosts
= argv_split(hosts
, " ,\t\r\n");
641 if (dict_mysql
->hosts
->argc
== 0) {
642 argv_add(dict_mysql
->hosts
, "localhost", ARGV_END
);
643 argv_terminate(dict_mysql
->hosts
);
645 msg_info("%s: %s: no hostnames specified, defaulting to '%s'",
646 myname
, mysqlcf
, dict_mysql
->hosts
->argv
[0]);
651 /* dict_mysql_open - open MYSQL data base */
653 DICT
*dict_mysql_open(const char *name
, int open_flags
, int dict_flags
)
655 DICT_MYSQL
*dict_mysql
;
660 if (open_flags
!= O_RDONLY
)
661 msg_fatal("%s:%s map requires O_RDONLY access mode",
662 DICT_TYPE_MYSQL
, name
);
664 dict_mysql
= (DICT_MYSQL
*) dict_alloc(DICT_TYPE_MYSQL
, name
,
666 dict_mysql
->dict
.lookup
= dict_mysql_lookup
;
667 dict_mysql
->dict
.close
= dict_mysql_close
;
668 dict_mysql
->dict
.flags
= dict_flags
;
669 mysql_parse_config(dict_mysql
, name
);
670 #if defined(MYSQL_VERSION_ID) && MYSQL_VERSION_ID >= 40000
671 dict_mysql
->active_host
= 0;
673 dict_mysql
->pldb
= plmysql_init(dict_mysql
->hosts
);
674 if (dict_mysql
->pldb
== NULL
)
675 msg_fatal("couldn't intialize pldb!\n");
676 return (DICT_DEBUG (&dict_mysql
->dict
));
680 * plmysql_init - initalize a MYSQL database.
681 * Return NULL on failure, or a PLMYSQL * on success.
683 static PLMYSQL
*plmysql_init(ARGV
*hosts
)
688 if ((PLDB
= (PLMYSQL
*) mymalloc(sizeof(PLMYSQL
))) == 0)
689 msg_fatal("mymalloc of pldb failed");
691 PLDB
->len_hosts
= hosts
->argc
;
692 if ((PLDB
->db_hosts
= (HOST
**) mymalloc(sizeof(HOST
*) * hosts
->argc
)) == 0)
694 for (i
= 0; i
< hosts
->argc
; i
++)
695 PLDB
->db_hosts
[i
] = host_init(hosts
->argv
[i
]);
701 /* host_init - initialize HOST structure */
702 static HOST
*host_init(const char *hostname
)
704 const char *myname
= "mysql host_init";
705 HOST
*host
= (HOST
*) mymalloc(sizeof(HOST
));
706 const char *d
= hostname
;
710 host
->hostname
= mystrdup(hostname
);
712 host
->stat
= STATUNTRIED
;
716 * Ad-hoc parsing code. Expect "unix:pathname" or "inet:host:port", where
717 * both "inet:" and ":port" are optional.
719 if (strncmp(d
, "unix:", 5) == 0) {
721 host
->type
= TYPEUNIX
;
723 if (strncmp(d
, "inet:", 5) == 0)
725 host
->type
= TYPEINET
;
727 host
->name
= mystrdup(d
);
728 if ((s
= split_at_right(host
->name
, ':')) != 0)
729 host
->port
= ntohs(find_inet_port(s
, "tcp"));
730 if (strcasecmp(host
->name
, "localhost") == 0) {
731 /* The MySQL way: this will actually connect over the UNIX socket */
734 host
->type
= TYPEUNIX
;
738 msg_info("%s: host=%s, port=%d, type=%s", myname
,
739 host
->name
? host
->name
: "localhost",
740 host
->port
, host
->type
== TYPEUNIX
? "unix" : "inet");
744 /* dict_mysql_close - close MYSQL database */
746 static void dict_mysql_close(DICT
*dict
)
749 DICT_MYSQL
*dict_mysql
= (DICT_MYSQL
*) dict
;
751 plmysql_dealloc(dict_mysql
->pldb
);
752 cfg_parser_free(dict_mysql
->parser
);
753 myfree(dict_mysql
->username
);
754 myfree(dict_mysql
->password
);
755 myfree(dict_mysql
->dbname
);
756 myfree(dict_mysql
->query
);
757 myfree(dict_mysql
->result_format
);
758 if (dict_mysql
->hosts
)
759 argv_free(dict_mysql
->hosts
);
761 db_common_free_ctx(dict_mysql
->ctx
);
763 vstring_free(dict
->fold_buf
);
767 /* plmysql_dealloc - free memory associated with PLMYSQL close databases */
768 static void plmysql_dealloc(PLMYSQL
*PLDB
)
772 for (i
= 0; i
< PLDB
->len_hosts
; i
++) {
773 event_cancel_timer(dict_mysql_event
, (char *) (PLDB
->db_hosts
[i
]));
774 if (PLDB
->db_hosts
[i
]->db
)
775 mysql_close(PLDB
->db_hosts
[i
]->db
);
776 myfree(PLDB
->db_hosts
[i
]->hostname
);
777 if (PLDB
->db_hosts
[i
]->name
)
778 myfree(PLDB
->db_hosts
[i
]->name
);
779 myfree((char *) PLDB
->db_hosts
[i
]);
781 myfree((char *) PLDB
->db_hosts
);
782 myfree((char *) (PLDB
));