utils: Fix up 14a533680245
[samba4-gss.git] / source3 / modules / vfs_preopen.c
blobaa37158cf20e52a733640deb913132dbce6ae83f
1 /*
2 * Force a readahead of files by opening them and reading the first bytes
4 * Copyright (C) Volker Lendecke 2008
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21 #include "includes.h"
22 #include "system/filesys.h"
23 #include "smbd/smbd.h"
24 #include "lib/util/sys_rw.h"
25 #include "lib/util/sys_rw_data.h"
26 #include "lib/util/smb_strtox.h"
27 #include "lib/util_matching.h"
28 #include "lib/global_contexts.h"
30 static int vfs_preopen_debug_level = DBGC_VFS;
32 #undef DBGC_CLASS
33 #define DBGC_CLASS vfs_preopen_debug_level
35 #define PREOPEN_MAX_DIGITS 19
36 #define PREOPEN_MAX_NUMBER (uint64_t)9999999999999999999ULL
38 struct preopen_state;
40 struct preopen_helper {
41 struct preopen_state *state;
42 struct tevent_fd *fde;
43 pid_t pid;
44 int fd;
45 bool busy;
48 struct preopen_state {
49 int num_helpers;
50 struct preopen_helper *helpers;
52 size_t to_read; /* How many bytes to read in children? */
53 int queue_max;
55 int queue_dbglvl; /* DBGLVL_DEBUG by default */
56 int nomatch_dbglvl; /* DBGLVL_INFO by default */
57 int match_dbglvl; /* DBGLVL_INFO by default */
58 int reset_dbglvl; /* DBGLVL_INFO by default */
59 int nodigits_dbglvl; /* DBGLVL_WARNING by default */
60 int founddigits_dbglvl; /* DBGLVL_NOTICE by default */
61 int push_dbglvl; /* DBGLVL_NOTICE by default */
63 char *template_fname; /* Filename to be sent to children */
64 size_t number_start; /* start offset into "template_fname" */
65 int num_digits; /* How many digits is the number long? */
67 uint64_t fnum_sent; /* last fname sent to children */
69 uint64_t fnum_queue_end;/* last fname to be sent, based on
70 * last open call + preopen:queuelen
73 struct samba_path_matching *preopen_names;
74 ssize_t last_match_idx; /* remember the last match */
77 static void preopen_helper_destroy(struct preopen_helper *c)
79 int status;
80 TALLOC_FREE(c->fde);
81 close(c->fd);
82 c->fd = -1;
83 kill(c->pid, SIGKILL);
84 waitpid(c->pid, &status, 0);
85 c->busy = true;
88 static void preopen_queue_run(struct preopen_state *state)
90 char *pdelimiter;
91 char delimiter;
93 DBG_PREFIX(state->queue_dbglvl, ("START: "
94 "last_fname[%s] start_offset=%zu num_digits=%d "
95 "last_pushed_num=%"PRIu64" queue_end_num=%"PRIu64" num_helpers=%d\n",
96 state->template_fname,
97 state->number_start,
98 state->num_digits,
99 state->fnum_sent,
100 state->fnum_queue_end,
101 state->num_helpers));
103 pdelimiter = state->template_fname + state->number_start
104 + state->num_digits;
105 delimiter = *pdelimiter;
107 while (state->fnum_sent < state->fnum_queue_end) {
109 ssize_t written;
110 size_t to_write;
111 int helper;
113 for (helper=0; helper<state->num_helpers; helper++) {
114 if (state->helpers[helper].busy) {
115 continue;
117 break;
119 if (helper == state->num_helpers) {
120 /* everyone is busy */
121 DBG_PREFIX(state->queue_dbglvl, ("BUSY: "
122 "template_fname[%s] start_offset=%zu num_digits=%d "
123 "last_pushed_num=%"PRIu64" queue_end_num=%"PRIu64"\n",
124 state->template_fname,
125 state->number_start,
126 state->num_digits,
127 state->fnum_sent,
128 state->fnum_queue_end));
129 return;
132 snprintf(state->template_fname + state->number_start,
133 state->num_digits + 1,
134 "%.*llu", state->num_digits,
135 (long long unsigned int)(state->fnum_sent + 1));
136 *pdelimiter = delimiter;
138 DBG_PREFIX(state->push_dbglvl, (
139 "PUSH: fullpath[%s] to helper(idx=%d)\n",
140 state->template_fname, helper));
142 to_write = talloc_get_size(state->template_fname);
143 written = write_data(state->helpers[helper].fd,
144 state->template_fname, to_write);
145 state->helpers[helper].busy = true;
147 if (written != to_write) {
148 preopen_helper_destroy(&state->helpers[helper]);
150 state->fnum_sent += 1;
152 DBG_PREFIX(state->queue_dbglvl, ("END: "
153 "template_fname[%s] start_offset=%zu num_digits=%d "
154 "last_pushed_num=%"PRIu64" queue_end_num=%"PRIu64"\n",
155 state->template_fname,
156 state->number_start,
157 state->num_digits,
158 state->fnum_sent,
159 state->fnum_queue_end));
162 static void preopen_helper_readable(struct tevent_context *ev,
163 struct tevent_fd *fde, uint16_t flags,
164 void *priv)
166 struct preopen_helper *helper = (struct preopen_helper *)priv;
167 struct preopen_state *state = helper->state;
168 ssize_t nread;
169 char c;
171 if ((flags & TEVENT_FD_READ) == 0) {
172 return;
175 nread = read(helper->fd, &c, 1);
176 if (nread <= 0) {
177 preopen_helper_destroy(helper);
178 return;
181 helper->busy = false;
183 DBG_PREFIX(state->queue_dbglvl, ("BEFORE: preopen_queue_run\n"));
184 preopen_queue_run(state);
185 DBG_PREFIX(state->queue_dbglvl, ("AFTER: preopen_queue_run\n"));
188 static int preopen_helpers_destructor(struct preopen_state *c)
190 int i;
192 for (i=0; i<c->num_helpers; i++) {
193 if (c->helpers[i].fd == -1) {
194 continue;
196 preopen_helper_destroy(&c->helpers[i]);
199 return 0;
202 static bool preopen_helper_open_one(int sock_fd, char **pnamebuf,
203 size_t to_read, void *filebuf)
205 char *namebuf = *pnamebuf;
206 size_t nread = 0;
207 ssize_t chunk;
208 char c = 0;
209 int fd;
211 do {
212 chunk = read(sock_fd, namebuf + nread,
213 talloc_get_size(namebuf) - nread);
214 if (chunk <= 0) {
215 return false;
218 if (nread + chunk < nread) {
219 return false;
221 nread += chunk;
223 if (nread == talloc_get_size(namebuf)) {
224 namebuf = talloc_realloc(
225 NULL, namebuf, char,
226 talloc_get_size(namebuf) * 2);
227 if (namebuf == NULL) {
228 return false;
230 *pnamebuf = namebuf;
232 } while (namebuf[nread - 1] != '\0');
234 fd = open(namebuf, O_RDONLY);
235 if (fd == -1) {
236 goto done;
238 chunk = read(fd, filebuf, to_read);
239 close(fd);
241 done:
242 sys_write_v(sock_fd, &c, 1);
243 return true;
246 static bool preopen_helper(int fd, size_t to_read)
248 char *namebuf;
249 void *readbuf;
251 namebuf = talloc_array(NULL, char, 1024);
252 if (namebuf == NULL) {
253 return false;
256 readbuf = talloc_size(NULL, to_read);
257 if (readbuf == NULL) {
258 TALLOC_FREE(namebuf);
259 return false;
262 while (preopen_helper_open_one(fd, &namebuf, to_read, readbuf)) {
266 TALLOC_FREE(readbuf);
267 TALLOC_FREE(namebuf);
268 return false;
271 static NTSTATUS preopen_init_helper(struct preopen_helper *h)
273 int fdpair[2];
274 NTSTATUS status;
276 if (socketpair(AF_UNIX, SOCK_STREAM, 0, fdpair) == -1) {
277 status = map_nt_error_from_unix(errno);
278 DEBUG(10, ("socketpair() failed: %s\n", strerror(errno)));
279 return status;
282 h->pid = fork();
284 if (h->pid == -1) {
285 return map_nt_error_from_unix(errno);
288 if (h->pid == 0) {
289 close(fdpair[0]);
290 preopen_helper(fdpair[1], h->state->to_read);
291 exit(0);
293 close(fdpair[1]);
294 h->fd = fdpair[0];
295 h->fde = tevent_add_fd(global_event_context(), h->state, h->fd,
296 TEVENT_FD_READ, preopen_helper_readable, h);
297 if (h->fde == NULL) {
298 close(h->fd);
299 h->fd = -1;
300 return NT_STATUS_NO_MEMORY;
302 h->busy = false;
303 return NT_STATUS_OK;
306 static NTSTATUS preopen_init_helpers(TALLOC_CTX *mem_ctx, size_t to_read,
307 int num_helpers, int queue_max,
308 struct preopen_state **presult)
310 struct preopen_state *result;
311 int i;
313 result = talloc(mem_ctx, struct preopen_state);
314 if (result == NULL) {
315 return NT_STATUS_NO_MEMORY;
318 result->num_helpers = num_helpers;
319 result->helpers = talloc_array(result, struct preopen_helper,
320 num_helpers);
321 if (result->helpers == NULL) {
322 TALLOC_FREE(result);
323 return NT_STATUS_NO_MEMORY;
326 result->to_read = to_read;
327 result->queue_max = queue_max;
328 result->template_fname = NULL;
329 result->fnum_sent = 0;
330 result->fnum_queue_end = 0;
332 for (i=0; i<num_helpers; i++) {
333 result->helpers[i].state = result;
334 result->helpers[i].fd = -1;
337 talloc_set_destructor(result, preopen_helpers_destructor);
339 for (i=0; i<num_helpers; i++) {
340 preopen_init_helper(&result->helpers[i]);
343 *presult = result;
344 return NT_STATUS_OK;
347 static void preopen_free_helpers(void **ptr)
349 TALLOC_FREE(*ptr);
352 static struct preopen_state *preopen_state_get(vfs_handle_struct *handle)
354 struct preopen_state *state;
355 NTSTATUS status;
356 const char *namelist;
358 if (SMB_VFS_HANDLE_TEST_DATA(handle)) {
359 SMB_VFS_HANDLE_GET_DATA(handle, state, struct preopen_state,
360 return NULL);
361 return state;
364 namelist = lp_parm_const_string(SNUM(handle->conn), "preopen", "names",
365 NULL);
367 if (namelist == NULL) {
368 return NULL;
371 status = preopen_init_helpers(
372 NULL,
373 lp_parm_int(SNUM(handle->conn), "preopen", "num_bytes", 1),
374 lp_parm_int(SNUM(handle->conn), "preopen", "helpers", 1),
375 lp_parm_int(SNUM(handle->conn), "preopen", "queuelen", 10),
376 &state);
377 if (!NT_STATUS_IS_OK(status)) {
378 return NULL;
381 state->queue_dbglvl = lp_parm_int(SNUM(handle->conn), "preopen", "queue_log_level", DBGLVL_DEBUG);
382 state->nomatch_dbglvl = lp_parm_int(SNUM(handle->conn), "preopen", "nomatch_log_level", DBGLVL_INFO);
383 state->match_dbglvl = lp_parm_int(SNUM(handle->conn), "preopen", "match_log_level", DBGLVL_INFO);
384 state->reset_dbglvl = lp_parm_int(SNUM(handle->conn), "preopen", "reset_log_level", DBGLVL_INFO);
385 state->nodigits_dbglvl = lp_parm_int(SNUM(handle->conn), "preopen", "nodigits_log_level", DBGLVL_WARNING);
386 state->founddigits_dbglvl = lp_parm_int(SNUM(handle->conn), "preopen", "founddigits_log_level", DBGLVL_NOTICE);
387 state->push_dbglvl = lp_parm_int(SNUM(handle->conn), "preopen", "push_log_level", DBGLVL_NOTICE);
389 if (lp_parm_bool(SNUM(handle->conn), "preopen", "posix-basic-regex", false)) {
390 status = samba_path_matching_regex_sub1_create(state,
391 namelist,
392 &state->preopen_names);
393 } else {
394 status = samba_path_matching_mswild_create(state,
395 true, /* case_sensitive */
396 namelist,
397 &state->preopen_names);
399 if (!NT_STATUS_IS_OK(status)) {
400 TALLOC_FREE(state);
401 return NULL;
403 state->last_match_idx = -1;
405 if (!SMB_VFS_HANDLE_TEST_DATA(handle)) {
406 SMB_VFS_HANDLE_SET_DATA(handle, state, preopen_free_helpers,
407 struct preopen_state, return NULL);
410 return state;
413 static bool preopen_parse_fname(const char *fname, uint64_t *pnum,
414 size_t *pstart_idx, int *pnum_digits)
416 char digits[PREOPEN_MAX_DIGITS+1] = { 0, };
417 const char *p;
418 char *q = NULL;
419 unsigned long long num;
420 size_t start_idx = 0;
421 int num_digits = -1;
422 int error = 0;
424 if (*pstart_idx > 0 && *pnum_digits > 0) {
426 * If the caller knowns
427 * how many digits are expected
428 * and on what position,
429 * we should copy the exact
430 * subset before we start
431 * parsing the string into a number
434 if (*pnum_digits > PREOPEN_MAX_DIGITS) {
436 * a string with as much digits as
437 * PREOPEN_MAX_DIGITS is the longest
438 * string that would make any sense for us.
440 * The rest will be checked via
441 * smb_strtoull().
443 return false;
445 p = fname + *pstart_idx;
446 memcpy(digits, p, *pnum_digits);
447 p = digits;
448 start_idx = *pstart_idx;
449 goto parse;
452 p = strrchr_m(fname, '/');
453 if (p == NULL) {
454 p = fname;
457 p += 1;
458 while (p[0] != '\0') {
459 if (isdigit(p[0]) && isdigit(p[1]) && isdigit(p[2])) {
460 break;
462 p += 1;
464 if (*p == '\0') {
465 /* no digits around */
466 return false;
469 start_idx = (p - fname);
471 parse:
472 num = smb_strtoull(p, (char **)&q, 10, &error, SMB_STR_STANDARD);
473 if (error != 0) {
474 return false;
477 if (num >= PREOPEN_MAX_NUMBER) {
478 /* overflow */
479 return false;
482 num_digits = (q - p);
484 if (*pnum_digits != -1 && *pnum_digits != num_digits) {
486 * If the caller knowns how many digits
487 * it expects we should fail if we got something
488 * different.
490 return false;
493 *pnum = num;
494 *pstart_idx = start_idx;
495 *pnum_digits = num_digits;
496 return true;
499 static uint64_t num_digits_max_value(int num_digits)
501 uint64_t num_max = 1;
502 int i;
504 if (num_digits < 1) {
505 return 0;
507 if (num_digits >= PREOPEN_MAX_DIGITS) {
508 return PREOPEN_MAX_NUMBER;
511 for (i = 0; i < num_digits; i++) {
512 num_max *= 10;
516 * We actually want
517 * 9 instead of 10
518 * 99 instead of 100
519 * 999 instead of 1000
521 return num_max - 1;
524 static int preopen_openat(struct vfs_handle_struct *handle,
525 const struct files_struct *dirfsp,
526 const struct smb_filename *smb_fname,
527 struct files_struct *fsp,
528 const struct vfs_open_how *how)
530 const char *dirname = dirfsp->fsp_name->base_name;
531 struct preopen_state *state;
532 int res;
533 uint64_t num;
534 uint64_t num_max;
535 NTSTATUS status;
536 char *new_template = NULL;
537 size_t new_start = 0;
538 int new_digits = -1;
539 size_t new_end = 0;
540 ssize_t match_idx = -1;
541 ssize_t replace_start = -1;
542 ssize_t replace_end = -1;
543 bool need_reset = false;
545 DBG_DEBUG("called on %s\n", smb_fname_str_dbg(smb_fname));
547 state = preopen_state_get(handle);
548 if (state == NULL) {
549 return SMB_VFS_NEXT_OPENAT(handle,
550 dirfsp,
551 smb_fname,
552 fsp,
553 how);
556 res = SMB_VFS_NEXT_OPENAT(handle, dirfsp, smb_fname, fsp, how);
557 if (res == -1) {
558 return -1;
561 if ((how->flags & O_ACCMODE) != O_RDONLY) {
562 return res;
566 * Make sure we can later construct an absolute pathname
568 if (dirname[0] != '/') {
569 return res;
572 * There's no point in preopen the directory itself.
574 if (ISDOT(smb_fname->base_name)) {
575 return res;
578 * If we got an absolute path in
579 * smb_fname it's most likely the
580 * reopen via /proc/self/fd/$fd
582 if (smb_fname->base_name[0] == '/') {
583 return res;
586 status = samba_path_matching_check_last_component(state->preopen_names,
587 smb_fname->base_name,
588 &match_idx,
589 &replace_start,
590 &replace_end);
591 if (!NT_STATUS_IS_OK(status)) {
592 match_idx = -1;
594 if (match_idx < 0) {
595 DBG_PREFIX(state->nomatch_dbglvl, (
596 "No match with the preopen:names list by name[%s]\n",
597 smb_fname_str_dbg(smb_fname)));
598 return res;
601 if (replace_start != -1 && replace_end != -1) {
602 DBG_PREFIX(state->match_dbglvl, (
603 "Pattern(idx=%zd) from preopen:names list matched name[%s] hints(start=%zd,end=%zd)\n",
604 match_idx, smb_fname_str_dbg(smb_fname), replace_start, replace_end));
605 } else {
606 DBG_PREFIX(state->match_dbglvl, (
607 "Pattern(idx=%zd) from preopen:names list matched name[%s]\n",
608 match_idx, smb_fname_str_dbg(smb_fname)));
611 new_template = talloc_asprintf(
612 state, "%s/%s",
613 dirname, smb_fname->base_name);
614 if (new_template == NULL) {
615 DBG_ERR("talloc_asprintf(%s/%s) failed\n",
616 dirname, smb_fname_str_dbg(smb_fname));
617 return res;
620 if (replace_start != -1 && replace_end != -1) {
621 size_t dirofs = strlen(dirname) + 1;
622 new_start = dirofs + replace_start;
623 new_digits = replace_end - replace_start;
626 if (!preopen_parse_fname(new_template, &num,
627 &new_start, &new_digits)) {
628 DBG_PREFIX(state->nodigits_dbglvl, (
629 "Pattern(idx=%zd) no valid digits found on fullpath[%s]\n",
630 match_idx, new_template));
631 TALLOC_FREE(new_template);
632 return res;
634 new_end = new_start + new_digits;
636 DBG_PREFIX(state->founddigits_dbglvl, (
637 "Pattern(idx=%zd) found num_digits[%d] start_offset[%zd] parsed_num[%"PRIu64"] fullpath[%s]\n",
638 match_idx, new_digits, new_start, num, new_template));
640 if (state->last_match_idx != match_idx) {
642 * If a different pattern caused the match
643 * we better reset the queue
645 if (state->last_match_idx != -1) {
646 DBG_PREFIX(state->reset_dbglvl, ("RESET: "
647 "pattern changed from idx=%zd to idx=%zd by fullpath[%s]\n",
648 state->last_match_idx, match_idx, new_template));
650 need_reset = true;
651 } else if (state->number_start != new_start) {
653 * If the digits started at a different position
654 * we better reset the queue
656 DBG_PREFIX(state->reset_dbglvl, ("RESET: "
657 "start_offset changed from byte=%zd to byte=%zd by fullpath[%s]\n",
658 state->number_start, new_start, new_template));
659 need_reset = true;
660 } else if (state->num_digits != new_digits) {
662 * If number of digits changed
663 * we better reset the queue
665 DBG_PREFIX(state->reset_dbglvl, ("RESET: "
666 "num_digits changed %d to %d by fullpath[%s]\n",
667 state->num_digits, new_digits, new_template));
668 need_reset = true;
669 } else if (strncmp(state->template_fname, new_template, new_start) != 0) {
671 * If name before the digits changed
672 * we better reset the queue
674 DBG_PREFIX(state->reset_dbglvl, ("RESET: "
675 "leading pathprefix[%.*s] changed by fullpath[%s]\n",
676 (int)state->number_start, state->template_fname, new_template));
677 need_reset = true;
678 } else if (strcmp(state->template_fname + new_end, new_template + new_end) != 0) {
680 * If name after the digits changed
681 * we better reset the queue
683 DBG_PREFIX(state->reset_dbglvl, ("RESET: "
684 "trailing suffix[%s] changed by fullpath[%s]\n",
685 state->template_fname + new_end, new_template));
686 need_reset = true;
689 if (need_reset) {
691 * Reset the queue
693 state->fnum_sent = 0;
694 state->fnum_queue_end = 0;
695 state->last_match_idx = match_idx;
698 TALLOC_FREE(state->template_fname);
699 state->template_fname = new_template;
700 state->number_start = new_start;
701 state->num_digits = new_digits;
703 if (num > state->fnum_sent) {
705 * Helpers were too slow, there's no point in reading
706 * files in helpers that we already read in the
707 * parent.
709 state->fnum_sent = num;
712 if ((state->fnum_queue_end != 0) /* Something was started earlier */
713 && (num < (state->fnum_queue_end - state->queue_max))) {
715 * "num" is before the queue we announced. This means
716 * a new run is started.
718 state->fnum_sent = num;
721 num_max = num_digits_max_value(state->num_digits);
722 state->fnum_queue_end = MIN(num_max, num + state->queue_max);
724 DBG_PREFIX(state->queue_dbglvl, ("BEFORE: preopen_queue_run\n"));
725 preopen_queue_run(state);
726 DBG_PREFIX(state->queue_dbglvl, ("AFTER: preopen_queue_run\n"));
728 return res;
731 static struct vfs_fn_pointers vfs_preopen_fns = {
732 .openat_fn = preopen_openat,
735 static_decl_vfs;
736 NTSTATUS vfs_preopen_init(TALLOC_CTX *ctx)
738 NTSTATUS status;
740 status = smb_register_vfs(SMB_VFS_INTERFACE_VERSION,
741 "preopen",
742 &vfs_preopen_fns);
743 if (!NT_STATUS_IS_OK(status)) {
744 return status;
747 vfs_preopen_debug_level = debug_add_class("preopen");
748 if (vfs_preopen_debug_level == -1) {
749 vfs_preopen_debug_level = DBGC_VFS;
750 DBG_ERR("Couldn't register custom debugging class!\n");
751 } else {
752 DBG_DEBUG("Debug class number of 'preopen': %d\n",
753 vfs_preopen_debug_level);
756 return NT_STATUS_OK;