?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
updated to 64bit crc version of ccan crc
[httpd-crcsyncproxy.git] / crccache / mod_crccache_client.c
blobbcb682358d69840e56ae76685a251c1018f6a0b7
1 /* Licensed to the Apache Software Foundation (ASF) under one or more
2 * contributor license agreements. See the NOTICE file distributed with
3 * this work for additional information regarding copyright ownership.
4 * The ASF licenses this file to You under the Apache License, Version 2.0
5 * (the "License"); you may not use this file except in compliance with
6 * the License. You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 /* crcsync/crccache apache client module
18 *
19 * This module is designed to run as a cache server on the local end of a slow
20 * internet link. This module uses a crc32 running hash algorithm to reduce
21 * data transfer in cached but modified upstream files.
22 *
23 * CRC algorithm uses the crcsync library created by Rusty Russel
24 *
25 * Author: Toby Collett (2009)
26 * Contributor: Alex Wulms (2009)
27 *
28 */
29
30
31
32 #include <assert.h>
33
34 #include "apr_file_io.h"
35 #include "apr_strings.h"
36 #include "mod_cache.h"
37 #include "mod_disk_cache.h"
38 #include "ap_provider.h"
39 #include "util_filter.h"
40 #include "util_script.h"
41 #include "util_charset.h"
42
43 #include "crccache.h"
44 #include "ap_wrapper.h"
45 #include <crcsync/crcsync.h>
46 #include "zlib.h"
47
48 /*
49 * mod_disk_cache: Disk Based HTTP 1.1 Cache.
50 *
51 * Flow to Find the .data file:
52 * Incoming client requests URI /foo/bar/baz
53 * Generate <hash> off of /foo/bar/baz
54 * Open <hash>.header
55 * Read in <hash>.header file (may contain Format #1 or Format #2)
56 * If format #1 (Contains a list of Vary Headers):
57 * Use each header name (from .header) with our request values (headers_in) to
58 * regenerate <hash> using HeaderName+HeaderValue+.../foo/bar/baz
59 * re-read in <hash>.header (must be format #2)
60 * read in <hash>.data
61 *
62 * Format #1:
63 * apr_uint32_t format;
64 * apr_time_t expire;
65 * apr_array_t vary_headers (delimited by CRLF)
66 *
67 * Format #2:
68 * disk_cache_info_t (first sizeof(apr_uint32_t) bytes is the format)
69 * entity name (dobj->name) [length is in disk_cache_info_t->name_len]
70 * r->headers_out (delimited by CRLF)
71 * CRLF
72 * r->headers_in (delimited by CRLF)
73 * CRLF
74 */
75
76 module AP_MODULE_DECLARE_DATA crccache_client_module;
77
78 /* Forward declarations */
79 static int remove_entity(cache_handle_t *h);
80 static apr_status_t store_headers(cache_handle_t *h, request_rec *r,
81 cache_info *i);
82 static apr_status_t store_body(cache_handle_t *h, request_rec *r,
83 apr_bucket_brigade *b);
84 static apr_status_t recall_headers(cache_handle_t *h, request_rec *r);
85 static apr_status_t recall_body(cache_handle_t *h, apr_pool_t *p,
86 apr_bucket_brigade *bb);
87 static apr_status_t read_array(request_rec *r, apr_array_header_t* arr,
88 apr_file_t *file);
89
90 static ap_filter_rec_t *crccache_decode_filter_handle;
91
92 typedef enum decoding_state {
93 DECODING_NEW_SECTION,
94 DECODING_COMPRESSED,
95 DECODING_BLOCK_HEADER,
96 DECODING_BLOCK
97 } decoding_state;
98
99 typedef enum {
100 DECOMPRESSION_INITIALIZED,
101 DECOMPRESSION_ENDED
102 } decompression_state_t;
103
104 typedef struct crccache_client_ctx_t {
105 apr_bucket_brigade *bb;
106 size_t block_size;
107 size_t tail_block_size;
108 apr_bucket * cached_bucket;// original data so we can fill in the matched blocks
109
110 decoding_state state;
111 decompression_state_t decompression_state;
112 z_stream *decompression_stream;
113 int headers_checked;
114 } crccache_client_ctx;
115
116 /*
117 * Local static functions
118 */
119
120 static char *header_file(apr_pool_t *p, disk_cache_conf *conf,
121 disk_cache_object_t *dobj, const char *name) {
122 if (!dobj->hashfile) {
123 dobj->hashfile = ap_cache_generate_name(p, conf->dirlevels,
124 conf->dirlength, name);
125 }
126
127 if (dobj->prefix) {
128 return apr_pstrcat(p, dobj->prefix, CACHE_VDIR_SUFFIX, "/",
129 dobj->hashfile, CACHE_HEADER_SUFFIX, NULL);
130 } else {
131 return apr_pstrcat(p, conf->cache_root, "/", dobj->hashfile,
132 CACHE_HEADER_SUFFIX, NULL);
133 }
134 }
135
136 static char *data_file(apr_pool_t *p, disk_cache_conf *conf,
137 disk_cache_object_t *dobj, const char *name) {
138 if (!dobj->hashfile) {
139 dobj->hashfile = ap_cache_generate_name(p, conf->dirlevels,
140 conf->dirlength, name);
141 }
142
143 if (dobj->prefix) {
144 return apr_pstrcat(p, dobj->prefix, CACHE_VDIR_SUFFIX, "/",
145 dobj->hashfile, CACHE_DATA_SUFFIX, NULL);
146 } else {
147 return apr_pstrcat(p, conf->cache_root, "/", dobj->hashfile,
148 CACHE_DATA_SUFFIX, NULL);
149 }
150 }
151
152 static void mkdir_structure(disk_cache_conf *conf, const char *file,
153 apr_pool_t *pool) {
154 apr_status_t rv;
155 char *p;
156
157 for (p = (char*) file + conf->cache_root_len + 1;;) {
158 p = strchr(p, '/');
159 if (!p)
160 break;
161 *p = '\0';
162
163 rv = apr_dir_make(file, APR_UREAD | APR_UWRITE | APR_UEXECUTE, pool);
164 if (rv != APR_SUCCESS && !APR_STATUS_IS_EEXIST(rv)) {
165 /* XXX */
166 }
167 *p = '/';
168 ++p;
169 }
170 }
171
172 /* htcacheclean may remove directories underneath us.
173 * So, we'll try renaming three times at a cost of 0.002 seconds.
174 */
175 static apr_status_t safe_file_rename(disk_cache_conf *conf, const char *src,
176 const char *dest, apr_pool_t *pool) {
177 apr_status_t rv;
178
179 rv = apr_file_rename(src, dest, pool);
180
181 if (rv != APR_SUCCESS) {
182 int i;
183
184 for (i = 0; i < 2 && rv != APR_SUCCESS; i++) {
185 /* 1000 micro-seconds aka 0.001 seconds. */
186 apr_sleep(1000);
187
188 mkdir_structure(conf, dest, pool);
189
190 rv = apr_file_rename(src, dest, pool);
191 }
192 }
193
194 return rv;
195 }
196
197 static apr_status_t file_cache_el_final(disk_cache_object_t *dobj,
198 request_rec *r) {
199 /* move the data over */
200 if (dobj->tfd) {
201 apr_status_t rv;
202
203 apr_file_close(dobj->tfd);
204
205 /* This assumes that the tempfile is on the same file system
206 * as the cache_root. If not, then we need a file copy/move
207 * rather than a rename.
208 */
209 rv = apr_file_rename(dobj->tempfile, dobj->datafile, r->pool);
210 if (rv != APR_SUCCESS) {
211 ap_log_error(APLOG_MARK, APLOG_WARNING, rv,r->server, "disk_cache: rename tempfile to datafile failed:"
212 " %s -> %s", dobj->tempfile, dobj->datafile);
213 apr_file_remove(dobj->tempfile, r->pool);
214 }
215
216 dobj->tfd = NULL;
217 }
218
219 return APR_SUCCESS;
220 }
221
222 static apr_status_t file_cache_errorcleanup(disk_cache_object_t *dobj,
223 request_rec *r) {
224 /* Remove the header file and the body file. */
225 apr_file_remove(dobj->hdrsfile, r->pool);
226 apr_file_remove(dobj->datafile, r->pool);
227
228 /* If we opened the temporary data file, close and remove it. */
229 if (dobj->tfd) {
230 apr_file_close(dobj->tfd);
231 apr_file_remove(dobj->tempfile, r->pool);
232 dobj->tfd = NULL;
233 }
234
235 return APR_SUCCESS;
236 }
237
238 /* These two functions get and put state information into the data
239 * file for an ap_cache_el, this state information will be read
240 * and written transparent to clients of this module
241 */
242 static int file_cache_recall_mydata(apr_file_t *fd, cache_info *info,
243 disk_cache_object_t *dobj, request_rec *r) {
244 apr_status_t rv;
245 char *urlbuff;
246 disk_cache_info_t disk_info;
247 apr_size_t len;
248
249 /* read the data from the cache file */
250 len = sizeof(disk_cache_info_t);
251 rv = apr_file_read_full(fd, &disk_info, len, &len);
252 if (rv != APR_SUCCESS) {
253 return rv;
254 }
255
256 /* Store it away so we can get it later. */
257 dobj->disk_info = disk_info;
258
259 info->status = disk_info.status;
260 info->date = disk_info.date;
261 info->expire = disk_info.expire;
262 info->request_time = disk_info.request_time;
263 info->response_time = disk_info.response_time;
264
265 /* Note that we could optimize this by conditionally doing the palloc
266 * depending upon the size. */
267 urlbuff = apr_palloc(r->pool, disk_info.name_len + 1);
268 len = disk_info.name_len;
269 rv = apr_file_read_full(fd, urlbuff, len, &len);
270 if (rv != APR_SUCCESS) {
271 return rv;
272 }
273 urlbuff[disk_info.name_len] = '\0';
274
275 /* check that we have the same URL */
276 /* Would strncmp be correct? */
277 if (strcmp(urlbuff, dobj->name) != 0) {
278 return APR_EGENERAL;
279 }
280
281 return APR_SUCCESS;
282 }
283
284 static const char* regen_key(apr_pool_t *p, apr_table_t *headers,
285 apr_array_header_t *varray, const char *oldkey) {
286 struct iovec *iov;
287 int i, k;
288 int nvec;
289 const char *header;
290 const char **elts;
291
292 nvec = (varray->nelts * 2) + 1;
293 iov = apr_palloc(p, sizeof(struct iovec) * nvec);
294 elts = (const char **) varray->elts;
295
296 /* TODO:
297 * - Handle multiple-value headers better. (sort them?)
298 * - Handle Case in-sensitive Values better.
299 * This isn't the end of the world, since it just lowers the cache
300 * hit rate, but it would be nice to fix.
301 *
302 * The majority are case insenstive if they are values (encoding etc).
303 * Most of rfc2616 is case insensitive on header contents.
304 *
305 * So the better solution may be to identify headers which should be
306 * treated case-sensitive?
307 * HTTP URI's (3.2.3) [host and scheme are insensitive]
308 * HTTP method (5.1.1)
309 * HTTP-date values (3.3.1)
310 * 3.7 Media Types [exerpt]
311 * The type, subtype, and parameter attribute names are case-
312 * insensitive. Parameter values might or might not be case-sensitive,
313 * depending on the semantics of the parameter name.
314 * 4.20 Except [exerpt]
315 * Comparison of expectation values is case-insensitive for unquoted
316 * tokens (including the 100-continue token), and is case-sensitive for
317 * quoted-string expectation-extensions.
318 */
319
320 for (i = 0, k = 0; i < varray->nelts; i++) {
321 header = apr_table_get(headers, elts[i]);
322 if (!header) {
323 header = "";
324 }
325 iov[k].iov_base = (char*) elts[i];
326 iov[k].iov_len = strlen(elts[i]);
327 k++;
328 iov[k].iov_base = (char*) header;
329 iov[k].iov_len = strlen(header);
330 k++;
331 }
332 iov[k].iov_base = (char*) oldkey;
333 iov[k].iov_len = strlen(oldkey);
334 k++;
335
336 return apr_pstrcatv(p, iov, k, NULL);
337 }
338
339 static int array_alphasort(const void *fn1, const void *fn2) {
340 return strcmp(*(char**) fn1, *(char**) fn2);
341 }
342
343 static void tokens_to_array(apr_pool_t *p, const char *data,
344 apr_array_header_t *arr) {
345 char *token;
346
347 while ((token = ap_get_list_item(p, &data)) != NULL) {
348 *((const char **) apr_array_push(arr)) = token;
349 }
350
351 /* Sort it so that "Vary: A, B" and "Vary: B, A" are stored the same. */
352 qsort((void *) arr->elts, arr->nelts, sizeof(char *), array_alphasort);
353 }
354
355 /*
356 * Hook and mod_cache callback functions
357 */
358 static int create_entity(cache_handle_t *h, request_rec *r, const char *key,
359 apr_off_t len) {
360 disk_cache_conf *conf = ap_get_module_config(r->server->module_config,
361 &crccache_client_module);
362 cache_object_t *obj;
363 disk_cache_object_t *dobj;
364
365 if (conf->cache_root == NULL) {
366 return DECLINED;
367 }
368
369 /* Allocate and initialize cache_object_t and disk_cache_object_t */
370 h->cache_obj = obj = apr_pcalloc(r->pool, sizeof(*obj));
371 obj->vobj = dobj = apr_pcalloc(r->pool, sizeof(*dobj));
372
373 obj->key = apr_pstrdup(r->pool, key);
374
375 dobj->name = obj->key;
376 dobj->prefix = NULL;
377 /* Save the cache root */
378 dobj->root = apr_pstrndup(r->pool, conf->cache_root, conf->cache_root_len);
379 dobj->root_len = conf->cache_root_len;
380 dobj->datafile = data_file(r->pool, conf, dobj, key);
381 dobj->hdrsfile = header_file(r->pool, conf, dobj, key);
382 dobj->tempfile = apr_pstrcat(r->pool, conf->cache_root, AP_TEMPFILE, NULL);
383
384 return OK;
385 }
386
387 static int open_entity(cache_handle_t *h, request_rec *r, const char *key) {
388 apr_uint32_t format;
389 apr_size_t len;
390 const char *nkey;
391 apr_status_t rc;
392 static int error_logged = 0;
393 disk_cache_conf *conf = ap_get_module_config(r->server->module_config,
394 &crccache_client_module);
395 apr_finfo_t finfo;
396 cache_object_t *obj;
397 cache_info *info;
398 disk_cache_object_t *dobj;
399 int flags;
400 h->cache_obj = NULL;
401
402 /* Look up entity keyed to 'url' */
403 if (conf->cache_root == NULL) {
404 if (!error_logged) {
405 error_logged = 1;
406 ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server,
407 "disk_cache: Cannot cache files to disk without a CacheRootClient specified.");
408 }
409 return DECLINED;
410 }
411
412 /* Create and init the cache object */
413 h->cache_obj = obj = apr_pcalloc(r->pool, sizeof(cache_object_t));
414 obj->vobj = dobj = apr_pcalloc(r->pool, sizeof(disk_cache_object_t));
415
416 info = &(obj->info);
417
418 /* Open the headers file */
419 dobj->prefix = NULL;
420
421 /* Save the cache root */
422 dobj->root = apr_pstrndup(r->pool, conf->cache_root, conf->cache_root_len);
423 dobj->root_len = conf->cache_root_len;
424
425 dobj->hdrsfile = header_file(r->pool, conf, dobj, key);
426 flags = APR_READ|APR_BINARY|APR_BUFFERED;
427 rc = apr_file_open(&dobj->hfd, dobj->hdrsfile, flags, 0, r->pool);
428 if (rc != APR_SUCCESS) {
429 return DECLINED;
430 }
431
432 /* read the format from the cache file */
433 len = sizeof(format);
434 apr_file_read_full(dobj->hfd, &format, len, &len);
435
436 if (format == VARY_FORMAT_VERSION) {
437 apr_array_header_t* varray;
438 apr_time_t expire;
439
440 len = sizeof(expire);
441 apr_file_read_full(dobj->hfd, &expire, len, &len);
442
443 varray = apr_array_make(r->pool, 5, sizeof(char*));
444 rc = read_array(r, varray, dobj->hfd);
445 if (rc != APR_SUCCESS) {
446 ap_log_error(APLOG_MARK, APLOG_ERR, rc, r->server,
447 "disk_cache: Cannot parse vary header file: %s",
448 dobj->hdrsfile);
449 return DECLINED;
450 }
451 apr_file_close(dobj->hfd);
452
453 nkey = regen_key(r->pool, r->headers_in, varray, key);
454
455 dobj->hashfile = NULL;
456 dobj->prefix = dobj->hdrsfile;
457 dobj->hdrsfile = header_file(r->pool, conf, dobj, nkey);
458
459 flags = APR_READ|APR_BINARY|APR_BUFFERED;
460 rc = apr_file_open(&dobj->hfd, dobj->hdrsfile, flags, 0, r->pool);
461 if (rc != APR_SUCCESS) {
462 return DECLINED;
463 }
464 }
465 else if (format != DISK_FORMAT_VERSION) {
466 ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server,
467 "cache_disk: File '%s' has a version mismatch. File had version: %d.",
468 dobj->hdrsfile, format);
469 return DECLINED;
470 }
471 else {
472 apr_off_t offset = 0;
473 /* This wasn't a Vary Format file, so we must seek to the
474 * start of the file again, so that later reads work.
475 */
476 apr_file_seek(dobj->hfd, APR_SET, &offset);
477 nkey = key;
478 }
479
480 obj->key = nkey;
481 dobj->key = nkey;
482 dobj->name = key;
483 dobj->datafile = data_file(r->pool, conf, dobj, nkey);
484 dobj->tempfile = apr_pstrcat(r->pool, conf->cache_root, AP_TEMPFILE, NULL);
485
486 /* Open the data file */
487 flags = APR_READ|APR_BINARY;
488 #ifdef APR_SENDFILE_ENABLED
489 flags |= APR_SENDFILE_ENABLED;
490 #endif
491 rc = apr_file_open(&dobj->fd, dobj->datafile, flags, 0, r->pool);
492 if (rc != APR_SUCCESS) {
493 /* XXX: Log message */
494 return DECLINED;
495 }
496
497 rc = apr_file_info_get(&finfo, APR_FINFO_SIZE, dobj->fd);
498 if (rc == APR_SUCCESS) {
499 dobj->file_size = finfo.size;
500 }
501
502 /* Read the bytes to setup the cache_info fields */
503 rc = file_cache_recall_mydata(dobj->hfd, info, dobj, r);
504 if (rc != APR_SUCCESS) {
505 /* XXX log message */
506 return DECLINED;
507 }
508
509 /* Initialize the cache_handle callback functions */
510 ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
511 "disk_cache: Recalled cached URL info header %s", dobj->name);
512 return OK;
513 }
514
515 static int remove_entity(cache_handle_t *h) {
516 /* Null out the cache object pointer so next time we start from scratch */
517 h->cache_obj = NULL;
518
519 return OK;
520 }
521
522 static int remove_url(cache_handle_t *h, apr_pool_t *p) {
523 apr_status_t rc;
524 disk_cache_object_t *dobj;
525
526 /* Get disk cache object from cache handle */
527 dobj = (disk_cache_object_t *) h->cache_obj->vobj;
528 if (!dobj) {
529 return DECLINED;
530 }
531
532 /* Delete headers file */
533 if (dobj->hdrsfile) {
534 ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, NULL,
535 "disk_cache: Deleting %s from cache.", dobj->hdrsfile);
536
537 rc = apr_file_remove(dobj->hdrsfile, p);
538 if ((rc != APR_SUCCESS) && !APR_STATUS_IS_ENOENT(rc)) {
539 /* Will only result in an output if httpd is started with -e debug.
540 * For reason see log_error_core for the case s == NULL.
541 */
542 ap_log_error(APLOG_MARK, APLOG_DEBUG, rc, NULL,
543 "disk_cache: Failed to delete headers file %s from cache.",
544 dobj->hdrsfile);
545 return DECLINED;
546 }
547 }
548
549 /* Delete data file */
550 if (dobj->datafile) {
551 ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, NULL,
552 "disk_cache: Deleting %s from cache.", dobj->datafile);
553
554 rc = apr_file_remove(dobj->datafile, p);
555 if ((rc != APR_SUCCESS) && !APR_STATUS_IS_ENOENT(rc)) {
556 /* Will only result in an output if httpd is started with -e debug.
557 * For reason see log_error_core for the case s == NULL.
558 */
559 ap_log_error(APLOG_MARK, APLOG_DEBUG, rc, NULL,
560 "disk_cache: Failed to delete data file %s from cache.",
561 dobj->datafile);
562 return DECLINED;
563 }
564 }
565
566 /* now delete directories as far as possible up to our cache root */
567 if (dobj->root) {
568 const char *str_to_copy;
569
570 str_to_copy = dobj->hdrsfile ? dobj->hdrsfile : dobj->datafile;
571 if (str_to_copy) {
572 char *dir, *slash, *q;
573
574 dir = apr_pstrdup(p, str_to_copy);
575
576 /* remove filename */
577 slash = strrchr(dir, '/');
578 *slash = '\0';
579
580 /*
581 * now walk our way back to the cache root, delete everything
582 * in the way as far as possible
583 *
584 * Note: due to the way we constructed the file names in
585 * header_file and data_file, we are guaranteed that the
586 * cache_root is suffixed by at least one '/' which will be
587 * turned into a terminating null by this loop. Therefore,
588 * we won't either delete or go above our cache root.
589 */
590 for (q = dir + dobj->root_len; *q; ) {
591 ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, NULL,
592 "disk_cache: Deleting directory %s from cache",
593 dir);
594
595 rc = apr_dir_remove(dir, p);
596 if (rc != APR_SUCCESS && !APR_STATUS_IS_ENOENT(rc)) {
597 break;
598 }
599 slash = strrchr(q, '/');
600 *slash = '\0';
601 }
602 }
603 }
604
605 return OK;
606 }
607
608 static apr_status_t read_array(request_rec *r, apr_array_header_t* arr,
609 apr_file_t *file) {
610 char w[MAX_STRING_LEN];
611 int p;
612 apr_status_t rv;
613
614 while (1) {
615 rv = apr_file_gets(w, MAX_STRING_LEN - 1, file);
616 if (rv != APR_SUCCESS) {
617 ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r,
618 "Premature end of vary array.");
619 return rv;
620 }
621
622 p = strlen(w);
623 if (p> 0 && w[p - 1] == '\n') {
624 if (p> 1 && w[p - 2] == CR) {
625 w[p - 2] = '\0';
626 }
627 else {
628 w[p - 1] = '\0';
629 }
630 }
631
632 /* If we've finished reading the array, break out of the loop. */
633 if (w[0] == '\0') {
634 break;
635 }
636
637 *((const char **) apr_array_push(arr)) = apr_pstrdup(r->pool, w);
638 }
639
640 return APR_SUCCESS;
641 }
642
643 static apr_status_t store_array(apr_file_t *fd, apr_array_header_t* arr) {
644 int i;
645 apr_status_t rv;
646 struct iovec iov[2];
647 apr_size_t amt;
648 const char **elts;
649
650 elts = (const char **) arr->elts;
651
652 for (i = 0; i < arr->nelts; i++) {
653 iov[0].iov_base = (char*) elts[i];
654 iov[0].iov_len = strlen(elts[i]);
655 iov[1].iov_base = CRLF;
656 iov[1].iov_len = sizeof(CRLF) - 1;
657
658 rv = apr_file_writev(fd, (const struct iovec *) &iov, 2,
659 &amt);
660 if (rv != APR_SUCCESS) {
661 return rv;
662 }
663 }
664
665 iov[0].iov_base = CRLF;
666 iov[0].iov_len = sizeof(CRLF) - 1;
667
668 return apr_file_writev(fd, (const struct iovec *) &iov, 1,
669 &amt);
670 }
671
672 static apr_status_t read_table(cache_handle_t *handle, request_rec *r,
673 apr_table_t *table, apr_file_t *file) {
674 char w[MAX_STRING_LEN];
675 char *l;
676 int p;
677 apr_status_t rv;
678
679 while (1) {
680
681 /* ### What about APR_EOF? */
682 rv = apr_file_gets(w, MAX_STRING_LEN - 1, file);
683 if (rv != APR_SUCCESS) {
684 ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r,
685 "Premature end of cache headers.");
686 return rv;
687 }
688
689 /* Delete terminal (CR?)LF */
690
691 p = strlen(w);
692 /* Indeed, the host's '\n':
693 '\012' for UNIX; '\015' for MacOS; '\025' for OS/390
694 -- whatever the script generates.
695 */
696 if (p> 0 && w[p - 1] == '\n') {
697 if (p> 1 && w[p - 2] == CR) {
698 w[p - 2] = '\0';
699 }
700 else {
701 w[p - 1] = '\0';
702 }
703 }
704
705 /* If we've finished reading the headers, break out of the loop. */
706 if (w[0] == '\0') {
707 break;
708 }
709
710 #if APR_CHARSET_EBCDIC
711 /* Chances are that we received an ASCII header text instead of
712 * the expected EBCDIC header lines. Try to auto-detect:
713 */
714 if (!(l = strchr(w, ':'))) {
715 int maybeASCII = 0, maybeEBCDIC = 0;
716 unsigned char *cp, native;
717 apr_size_t inbytes_left, outbytes_left;
718
719 for (cp = w; *cp != '\0'; ++cp) {
720 native = apr_xlate_conv_byte(ap_hdrs_from_ascii, *cp);
721 if (apr_isprint(*cp) && !apr_isprint(native))
722 ++maybeEBCDIC;
723 if (!apr_isprint(*cp) && apr_isprint(native))
724 ++maybeASCII;
725 }
726 if (maybeASCII> maybeEBCDIC) {
727 ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server,
728 "CGI Interface Error: Script headers apparently ASCII: (CGI = %s)",
729 r->filename);
730 inbytes_left = outbytes_left = cp - w;
731 apr_xlate_conv_buffer(ap_hdrs_from_ascii,
732 w, &inbytes_left, w, &outbytes_left);
733 }
734 }
735 #endif /*APR_CHARSET_EBCDIC*/
736
737 /* if we see a bogus header don't ignore it. Shout and scream */
738 if (!(l = strchr(w, ':'))) {
739 return APR_EGENERAL;
740 }
741
742 *l++ = '\0';
743 while (*l && apr_isspace(*l)) {
744 ++l;
745 }
746
747 apr_table_add(table, w, l);
748 }
749
750 return APR_SUCCESS;
751 }
752
753 /**
754 * Clean-up memory used by helper libraries, that don't know about apr_palloc
755 * and that (probably) use classical malloc/free
756 */
757 static apr_status_t deflate_ctx_cleanup(void *data)
758 {
759 crccache_client_ctx *ctx = (crccache_client_ctx *)data;
760
761 if (ctx != NULL)
762 {
763 if (ctx->decompression_state != DECOMPRESSION_ENDED)
764 {
765 inflateEnd(ctx->decompression_stream);
766 ctx->decompression_state = DECOMPRESSION_ENDED;
767 }
768 }
769 return APR_SUCCESS;
770 }
771
772
773 /*
774 * Reads headers from a buffer and returns an array of headers.
775 * Returns NULL on file error
776 * This routine tries to deal with too long lines and continuation lines.
777 * @@@: XXX: FIXME: currently the headers are passed thru un-merged.
778 * Is that okay, or should they be collapsed where possible?
779 */
780 static apr_status_t recall_headers(cache_handle_t *h, request_rec *r) {
781 const char *data;
782 apr_size_t len;
783 apr_bucket *e;
784 unsigned i;
785 int z_RC;
786
787 disk_cache_object_t *dobj = (disk_cache_object_t *) h->cache_obj->vobj;
788
789
790 /* This case should not happen... */
791 if (!dobj->hfd) {
792 /* XXX log message */
793 return APR_NOTFOUND;
794 }
795
796 h->req_hdrs = apr_table_make(r->pool, 20);
797 h->resp_hdrs = apr_table_make(r->pool, 20);
798
799 /* Call routine to read the header lines/status line */
800 read_table(h, r, h->resp_hdrs, dobj->hfd);
801 read_table(h, r, h->req_hdrs, dobj->hfd);
802
803 // TODO: We only really want to add our block hashes if the cache is not fresh
804 // TODO: We could achieve that by adding a filter here on sending the request
805 // and then doing all of this in the filter 'JIT'
806 e = apr_bucket_file_create(dobj->fd, 0, (apr_size_t) dobj->file_size, r->pool,
807 r->connection->bucket_alloc);
808
809 /* read */
810 apr_bucket_read(e, &data, &len, APR_BLOCK_READ);
811
812 // this will be rounded down, but thats okay
813 size_t blocksize = len/FULL_BLOCK_COUNT;
814 size_t tail_block_size = len % FULL_BLOCK_COUNT;
815 size_t block_count_including_final_block = FULL_BLOCK_COUNT + (tail_block_size != 0);
816 // sanity check for very small files
817 if (blocksize> 4)
818 {
819 //ap_log_error(APLOG_MARK, APLOG_DEBUG, APR_SUCCESS, r->server,"crccache: %d blocks of %ld bytes",FULL_BLOCK_COUNT,blocksize);
820
821 crccache_client_ctx * ctx;
822 ctx = apr_pcalloc(r->pool, sizeof(*ctx));
823 ctx->bb = apr_brigade_create(r->pool, r->connection->bucket_alloc);
824 ctx->block_size = blocksize;
825 ctx->tail_block_size = tail_block_size;
826 ctx->state = DECODING_NEW_SECTION;
827 ctx->cached_bucket = e;
828
829 // Setup inflate for decompressing non-matched literal data
830 ctx->decompression_stream = apr_palloc(r->pool, sizeof(*(ctx->decompression_stream)));
831 ctx->decompression_stream->zalloc = Z_NULL;
832 ctx->decompression_stream->zfree = Z_NULL;
833 ctx->decompression_stream->opaque = Z_NULL;
834 ctx->decompression_stream->avail_in = 0;
835 ctx->decompression_stream->next_in = Z_NULL;
836 z_RC = inflateInit(ctx->decompression_stream);
837 if (z_RC != Z_OK)
838 {
839 ap_log_error(APLOG_MARK, APLOG_WARNING, 0, r->server,
840 "Can not initialize decompression engine, return code: %d", z_RC);
841 return APR_SUCCESS;
842 }
843 ctx->decompression_state = DECOMPRESSION_INITIALIZED;
844
845 // Register a cleanup function to cleanup internal libz resources
846 apr_pool_cleanup_register(r->pool, ctx, deflate_ctx_cleanup,
847 apr_pool_cleanup_null);
848
849 // All OK to go for the crcsync decoding: add the headers
850 // and set-up the decoding filter
851
852 // add one for base 64 overflow and null terminator
853 char hash_set[HASH_HEADER_SIZE+HASH_BASE64_SIZE_PADDING+1];
854 // use buffer to set block size first
855 snprintf(hash_set,HASH_HEADER_SIZE,"%zu",len);
856 apr_table_set(r->headers_in, FILE_SIZE_HEADER, hash_set);
857
858 uint64_t crcs[block_count_including_final_block];
859 crc_of_blocks(data, len, blocksize, HASH_SIZE, crcs);
860
861 for (i = 0; i < block_count_including_final_block;++i)
862 {
863 // encode the hash into base64;
864 encode_30bithash(crcs[i],&hash_set[i*HASH_BASE64_SIZE_TX]);
865 //ap_log_error(APLOG_MARK, APLOG_DEBUG, APR_SUCCESS, r->server,"crccache: block %d, hash %08X",i,crcs[i]);
866 }
867 //apr_bucket_delete(e);
868 apr_table_set(r->headers_in, BLOCK_HEADER, hash_set);
869
870 // TODO: do we want to cache the hashes here?
871 //ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server, "adding block-hashes header: %s",hash_set);
872
873 // we want to add a filter here so that we can decode the response.
874 // we need access to the original cached data when we get the response as
875 // we need that to fill in the matched blocks.
876 ap_add_output_filter_handle(crccache_decode_filter_handle,
877 ctx, r, r->connection);
878
879 // TODO: why is hfd file only closed in this case?
880 apr_file_close(dobj->hfd);
881 }
882 ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
883 "disk_cache: Recalled headers for URL %s", dobj->name);
884 return APR_SUCCESS;
885 }
886
887 static apr_status_t recall_body(cache_handle_t *h, apr_pool_t *p,
888 apr_bucket_brigade *bb) {
889 apr_bucket *e;
890 disk_cache_object_t *dobj = (disk_cache_object_t*) h->cache_obj->vobj;
891
892 e = apr_bucket_file_create(dobj->fd, 0, (apr_size_t) dobj->file_size, p,
893 bb->bucket_alloc);
894
895 APR_BRIGADE_INSERT_HEAD(bb, e);
896 e = apr_bucket_eos_create(bb->bucket_alloc);
897 APR_BRIGADE_INSERT_TAIL(bb, e);
898
899 return APR_SUCCESS;
900 }
901
902 static apr_status_t store_table(apr_file_t *fd, apr_table_t *table) {
903 int i;
904 apr_status_t rv;
905 struct iovec iov[4];
906 apr_size_t amt;
907 apr_table_entry_t *elts;
908
909 elts = (apr_table_entry_t *) apr_table_elts(table)->elts;
910 for (i = 0; i < apr_table_elts(table)->nelts; ++i) {
911 if (elts[i].key != NULL) {
912 iov[0].iov_base = elts[i].key;
913 iov[0].iov_len = strlen(elts[i].key);
914 iov[1].iov_base = ": ";
915 iov[1].iov_len = sizeof(": ") - 1;
916 iov[2].iov_base = elts[i].val;
917 iov[2].iov_len = strlen(elts[i].val);
918 iov[3].iov_base = CRLF;
919 iov[3].iov_len = sizeof(CRLF) - 1;
920
921 rv = apr_file_writev(fd, (const struct iovec *) &iov, 4,
922 &amt);
923 if (rv != APR_SUCCESS) {
924 return rv;
925 }
926 }
927 }
928 iov[0].iov_base = CRLF;
929 iov[0].iov_len = sizeof(CRLF) - 1;
930 rv = apr_file_writev(fd, (const struct iovec *) &iov, 1,
931 &amt);
932 return rv;
933 }
934
935 static apr_status_t store_headers(cache_handle_t *h, request_rec *r,
936 cache_info *info) {
937 disk_cache_conf *conf = ap_get_module_config(r->server->module_config,
938 &crccache_client_module);
939
940 apr_status_t rv;
941 apr_size_t amt;
942 disk_cache_object_t *dobj = (disk_cache_object_t*) h->cache_obj->vobj;
943
944 disk_cache_info_t disk_info;
945 struct iovec iov[2];
946
947 /* This is flaky... we need to manage the cache_info differently */
948 h->cache_obj->info = *info;
949
950 if (r->headers_out) {
951 const char *tmp;
952
953 tmp = apr_table_get(r->headers_out, "Vary");
954
955 if (tmp) {
956 apr_array_header_t* varray;
957 apr_uint32_t format = VARY_FORMAT_VERSION;
958
959 /* If we were initially opened as a vary format, rollback
960 * that internal state for the moment so we can recreate the
961 * vary format hints in the appropriate directory.
962 */
963 if (dobj->prefix) {
964 dobj->hdrsfile = dobj->prefix;
965 dobj->prefix = NULL;
966 }
967
968 mkdir_structure(conf, dobj->hdrsfile, r->pool);
969
970 rv = apr_file_mktemp(&dobj->tfd, dobj->tempfile,
971 APR_CREATE | APR_WRITE | APR_BINARY | APR_EXCL,
972 r->pool);
973
974 if (rv != APR_SUCCESS) {
975 return rv;
976 }
977
978 amt = sizeof(format);
979 apr_file_write(dobj->tfd, &format, &amt);
980
981 amt = sizeof(info->expire);
982 apr_file_write(dobj->tfd, &info->expire, &amt);
983
984 varray = apr_array_make(r->pool, 6, sizeof(char*));
985 tokens_to_array(r->pool, tmp, varray);
986
987 store_array(dobj->tfd, varray);
988
989 apr_file_close(dobj->tfd);
990
991 dobj->tfd = NULL;
992
993 rv = safe_file_rename(conf, dobj->tempfile, dobj->hdrsfile,
994 r->pool);
995 if (rv != APR_SUCCESS) {
996 ap_log_error(APLOG_MARK, APLOG_WARNING, rv, r->server,
997 "disk_cache: rename tempfile to varyfile failed: %s -> %s",
998 dobj->tempfile, dobj->hdrsfile);
999 apr_file_remove(dobj->tempfile, r->pool);
1000 return rv;
1001 }
1002
1003 dobj->tempfile = apr_pstrcat(r->pool, conf->cache_root, AP_TEMPFILE, NULL);
1004 tmp = regen_key(r->pool, r->headers_in, varray, dobj->name);
1005 dobj->prefix = dobj->hdrsfile;
1006 dobj->hashfile = NULL;
1007 dobj->datafile = data_file(r->pool, conf, dobj, tmp);
1008 dobj->hdrsfile = header_file(r->pool, conf, dobj, tmp);
1009 }
1010 }
1011
1012
1013 rv = apr_file_mktemp(&dobj->hfd, dobj->tempfile,
1014 APR_CREATE | APR_WRITE | APR_BINARY |
1015 APR_BUFFERED | APR_EXCL, r->pool);
1016
1017 if (rv != APR_SUCCESS) {
1018 return rv;
1019 }
1020
1021 disk_info.format = DISK_FORMAT_VERSION;
1022 disk_info.date = info->date;
1023 disk_info.expire = info->expire;
1024 disk_info.entity_version = dobj->disk_info.entity_version++;
1025 disk_info.request_time = info->request_time;
1026 disk_info.response_time = info->response_time;
1027 disk_info.status = info->status;
1028
1029 disk_info.name_len = strlen(dobj->name);
1030
1031 iov[0].iov_base = (void*)&disk_info;
1032 iov[0].iov_len = sizeof(disk_cache_info_t);
1033 iov[1].iov_base = (void*)dobj->name;
1034 iov[1].iov_len = disk_info.name_len;
1035
1036 rv = apr_file_writev(dobj->hfd, (const struct iovec *) &iov, 2, &amt);
1037 if (rv != APR_SUCCESS) {
1038 return rv;
1039 }
1040
1041 if (r->headers_out) {
1042 apr_table_t *headers_out;
1043
1044 headers_out = ap_cache_cacheable_hdrs_out(r->pool, r->headers_out,
1045 r->server);
1046
1047 if (!apr_table_get(headers_out, "Content-Type")
1048 && r->content_type) {
1049 apr_table_setn(headers_out, "Content-Type",
1050 ap_make_content_type(r, r->content_type));
1051 }
1052
1053 headers_out = apr_table_overlay(r->pool, headers_out,
1054 r->err_headers_out);
1055 rv = store_table(dobj->hfd, headers_out);
1056 if (rv != APR_SUCCESS) {
1057 return rv;
1058 }
1059 }
1060
1061 /* Parse the vary header and dump those fields from the headers_in. */
1062 /* FIXME: Make call to the same thing cache_select calls to crack Vary. */
1063 if (r->headers_in) {
1064 apr_table_t *headers_in;
1065
1066 headers_in = ap_cache_cacheable_hdrs_out(r->pool, r->headers_in,
1067 r->server);
1068 rv = store_table(dobj->hfd, headers_in);
1069 if (rv != APR_SUCCESS) {
1070 return rv;
1071 }
1072 }
1073
1074 apr_file_close(dobj->hfd); /* flush and close */
1075
1076 /* Remove old file with the same name. If remove fails, then
1077 * perhaps we need to create the directory tree where we are
1078 * about to write the new headers file.
1079 */
1080 rv = apr_file_remove(dobj->hdrsfile, r->pool);
1081 if (rv != APR_SUCCESS) {
1082 mkdir_structure(conf, dobj->hdrsfile, r->pool);
1083 }
1084
1085 rv = safe_file_rename(conf, dobj->tempfile, dobj->hdrsfile, r->pool);
1086 if (rv != APR_SUCCESS) {
1087 ap_log_error(APLOG_MARK, APLOG_WARNING, rv, r->server,
1088 "disk_cache: rename tempfile to hdrsfile failed: %s -> %s",
1089 dobj->tempfile, dobj->hdrsfile);
1090 apr_file_remove(dobj->tempfile, r->pool);
1091 return rv;
1092 }
1093
1094 dobj->tempfile = apr_pstrcat(r->pool, conf->cache_root, AP_TEMPFILE, NULL);
1095
1096 ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
1097 "disk_cache: Stored headers for URL %s", dobj->name);
1098 return APR_SUCCESS;
1099 }
1100
1101 static apr_status_t store_body(cache_handle_t *h, request_rec *r,
1102 apr_bucket_brigade *bb) {
1103 apr_bucket *e;
1104 apr_status_t rv;
1105
1106 disk_cache_object_t *dobj = (disk_cache_object_t *) h->cache_obj->vobj;
1107 disk_cache_conf *conf = ap_get_module_config(r->server->module_config,
1108 &crccache_client_module);
1109
1110 /* We write to a temp file and then atomically rename the file over
1111 * in file_cache_el_final().
1112 */
1113 if (!dobj->tfd) {
1114 rv = apr_file_mktemp(&dobj->tfd, dobj->tempfile, APR_CREATE | APR_WRITE
1115 | APR_BINARY | APR_BUFFERED | APR_EXCL, r->pool);
1116 if (rv != APR_SUCCESS) {
1117 return rv;
1118 }
1119 dobj->file_size = 0;
1120 }
1121
1122 for (e = APR_BRIGADE_FIRST(bb); e != APR_BRIGADE_SENTINEL(bb); e = APR_BUCKET_NEXT(e)) {
1123 const char *str;
1124 apr_size_t length, written;
1125 rv = apr_bucket_read(e, &str, &length, APR_BLOCK_READ);
1126 if (rv != APR_SUCCESS) {
1127 ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server,
1128 "cache_disk: Error when reading bucket for URL %s",
1129 h->cache_obj->key);
1130 /* Remove the intermediate cache file and return non-APR_SUCCESS */
1131 file_cache_errorcleanup(dobj, r);
1132 return rv;
1133 }
1134 rv = apr_file_write_full(dobj->tfd, str, length, &written);
1135 if (rv != APR_SUCCESS) {
1136 ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server,
1137 "cache_disk: Error when writing cache file for URL %s",
1138 h->cache_obj->key);
1139 /* Remove the intermediate cache file and return non-APR_SUCCESS */
1140 file_cache_errorcleanup(dobj, r);
1141 return rv;
1142 }
1143 dobj->file_size += written;
1144 if (dobj->file_size> conf->maxfs) {
1145 ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
1146 "cache_disk: URL %s failed the size check "
1147 "(%" APR_OFF_T_FMT " > %" APR_OFF_T_FMT ")",
1148 h->cache_obj->key, dobj->file_size, conf->maxfs);
1149 /* Remove the intermediate cache file and return non-APR_SUCCESS */
1150 file_cache_errorcleanup(dobj, r);
1151 return APR_EGENERAL;
1152 }
1153 }
1154
1155 /* Was this the final bucket? If yes, close the temp file and perform
1156 * sanity checks.
1157 */
1158 if (APR_BUCKET_IS_EOS(APR_BRIGADE_LAST(bb))) {
1159 if (r->connection->aborted || r->no_cache) {
1160 ap_log_error(APLOG_MARK, APLOG_INFO, 0, r->server,
1161 "disk_cache: Discarding body for URL %s "
1162 "because connection has been aborted.",
1163 h->cache_obj->key);
1164 /* Remove the intermediate cache file and return non-APR_SUCCESS */
1165 file_cache_errorcleanup(dobj, r);
1166 return APR_EGENERAL;
1167 }
1168 if (dobj->file_size < conf->minfs) {
1169 ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
1170 "cache_disk: URL %s failed the size check "
1171 "(%" APR_OFF_T_FMT " < %" APR_OFF_T_FMT ")",
1172 h->cache_obj->key, dobj->file_size, conf->minfs);
1173 /* Remove the intermediate cache file and return non-APR_SUCCESS */
1174 file_cache_errorcleanup(dobj, r);
1175 return APR_EGENERAL;
1176 }
1177
1178 /* All checks were fine. Move tempfile to final destination */
1179 /* Link to the perm file, and close the descriptor */
1180 file_cache_el_final(dobj, r);
1181 ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
1182 "disk_cache: Body for URL %s cached.", dobj->name);
1183 }
1184
1185 return APR_SUCCESS;
1186 }
1187
1188 /*
1189 * CACHE_DECODE filter
1190 * ----------------
1191 *
1192 * Deliver cached content (headers and body) up the stack.
1193 */
1194 static int crccache_decode_filter(ap_filter_t *f, apr_bucket_brigade *bb) {
1195 apr_bucket *e;
1196 request_rec *r = f->r;
1197 // TODO: set up context type struct
1198 crccache_client_ctx *ctx = f->ctx;
1199
1200 // if this is the first pass in decoding we should check the headers etc
1201 // and fix up those headers that we modified as part of the encoding
1202 if (ctx->headers_checked == 0)
1203 {
1204 ctx->headers_checked = 1;
1205
1206 ap_log_error(APLOG_MARK, APLOG_DEBUG, APR_SUCCESS, r->server,
1207 "CRCSYNC retuned status code (%d)", r->status);
1208
1209 // TODO: make this work if we have multiple encodings
1210 const char * content_encoding;
1211 content_encoding = apr_table_get(r->headers_out, ENCODING_HEADER);
1212 if (content_encoding == NULL || strcmp(CRCCACHE_ENCODING, content_encoding)
1213 != 0) {
1214 ap_log_error(APLOG_MARK, APLOG_DEBUG, APR_SUCCESS, r->server,
1215 "CRCSYNC not decoding, content encoding bad (%s)", content_encoding?content_encoding:"NULL");
1216 ap_remove_output_filter(f);
1217 return ap_pass_brigade(f->next, bb);
1218 }
1219 // TODO: Remove crcsync from the content encoding header
1220
1221 // TODO: we should only set the status back to 200 if there are no
1222 // other instance codings used
1223 r->status = 200;
1224 r->status_line = "200 OK";
1225
1226
1227 // TODO: Fix up the etag as well
1228 }
1229
1230
1231
1232 /* Do nothing if asked to filter nothing. */
1233 if (APR_BRIGADE_EMPTY(bb)) {
1234 return ap_pass_brigade(f->next, bb);
1235 }
1236
1237 /* We require that we have a context already, otherwise we dont have our cached file
1238 * to fill in the gaps with.
1239 */
1240 if (!ctx) {
1241 ap_log_error(APLOG_MARK, APLOG_DEBUG, APR_SUCCESS, r->server,
1242 "No context available %s", r->uri);
1243 ap_remove_output_filter(f);
1244 return ap_pass_brigade(f->next, bb);
1245 }
1246
1247 while (!APR_BRIGADE_EMPTY(bb))
1248 {
1249 const char *data;
1250 apr_size_t len;
1251
1252 e = APR_BRIGADE_FIRST(bb);
1253
1254 if (APR_BUCKET_IS_EOS(e)) {
1255
1256 /* Remove EOS from the old list, and insert into the new. */
1257 APR_BUCKET_REMOVE(e);
1258 APR_BRIGADE_INSERT_TAIL(ctx->bb, e);
1259
1260 /* This filter is done once it has served up its content */
1261 ap_remove_output_filter(f);
1262
1263 // TODO: check strong hash here
1264
1265 /* Okay, we've seen the EOS.
1266 * Time to pass it along down the chain.
1267 */
1268 return ap_pass_brigade(f->next, ctx->bb);
1269 }
1270
1271 if (APR_BUCKET_IS_FLUSH(e)) {
1272 apr_status_t rv;
1273
1274 /* Remove flush bucket from old brigade anf insert into the new. */
1275 APR_BUCKET_REMOVE(e);
1276 APR_BRIGADE_INSERT_TAIL(ctx->bb, e);
1277 rv = ap_pass_brigade(f->next, ctx->bb);
1278 if (rv != APR_SUCCESS) {
1279 return rv;
1280 }
1281 continue;
1282 }
1283
1284 if (APR_BUCKET_IS_METADATA(e)) {
1285 /*
1286 * Remove meta data bucket from old brigade and insert into the
1287 * new.
1288 */
1289 APR_BUCKET_REMOVE(e);
1290 APR_BRIGADE_INSERT_TAIL(ctx->bb, e);
1291 continue;
1292 }
1293
1294 /* read */
1295 apr_bucket_read(e, &data, &len, APR_BLOCK_READ);
1296 //ap_log_error_wrapper(APLOG_MARK, APLOG_DEBUG, APR_SUCCESS, r->server,"CRCSYNC-DECODE read %zd bytes",len);
1297
1298 apr_size_t consumed_bytes = 0;
1299 while (consumed_bytes < len)
1300 {
1301 //ap_log_error_wrapper(APLOG_MARK, APLOG_DEBUG, APR_SUCCESS, r->server,"CRCSYNC-DECODE remaining %zd bytes",len - consumed_bytes);
1302 // no guaruntee that our buckets line up with our encoding sections
1303 // so we need a processing state machine stored in our context
1304 switch (ctx->state)
1305 {
1306 case DECODING_NEW_SECTION:
1307 {
1308 // check if we have a compressed section or a block section
1309 if (data[consumed_bytes] == ENCODING_COMPRESSED)
1310 ctx->state = DECODING_COMPRESSED;
1311 else if (data[consumed_bytes] == ENCODING_BLOCK)
1312 ctx->state = DECODING_BLOCK_HEADER;
1313 else
1314 {
1315 ap_log_error(APLOG_MARK, APLOG_ERR, APR_SUCCESS, r->server,
1316 "CRCSYNC-DECODE, unknown section %d(%c)",data[consumed_bytes],data[consumed_bytes]);
1317 apr_brigade_cleanup(bb);
1318 return APR_EGENERAL;
1319 }
1320 //ap_log_error(APLOG_MARK, APLOG_DEBUG, APR_SUCCESS, r->server,"CRCSYNC-DECODE found a new section %d",ctx->state);
1321 consumed_bytes++;
1322 break;
1323 }
1324 case DECODING_BLOCK_HEADER:
1325 {
1326 unsigned char block_number = data[consumed_bytes];
1327 consumed_bytes++;
1328 ctx->state = DECODING_NEW_SECTION;
1329
1330 // TODO: Output the indicated block here
1331 size_t current_block_size = block_number < FULL_BLOCK_COUNT ? ctx->block_size : ctx->tail_block_size;
1332 ap_log_error_wrapper(APLOG_MARK, APLOG_DEBUG, APR_SUCCESS, r->server,
1333 "CRCSYNC-DECODE block section, block %d, size %zu" ,block_number, current_block_size);
1334
1335 char * buf = apr_palloc(r->pool, current_block_size);
1336 const char * source_data;
1337 size_t source_len;
1338 apr_bucket_read(ctx->cached_bucket, &source_data, &source_len, APR_BLOCK_READ);
1339 assert(block_number < (FULL_BLOCK_COUNT + (ctx->tail_block_size != 0)));
1340 memcpy(buf,&source_data[block_number*ctx->block_size],current_block_size);
1341 apr_bucket * b = apr_bucket_pool_create(buf, current_block_size, r->pool, f->c->bucket_alloc);
1342 APR_BRIGADE_INSERT_TAIL(ctx->bb, b);
1343 break;
1344 }
1345 case DECODING_COMPRESSED:
1346 {
1347 unsigned char decompressed_data_buf[30000];
1348 int z_RC;
1349 z_stream *strm = ctx->decompression_stream;
1350 strm->avail_in = len - consumed_bytes;
1351 strm->next_in = (Bytef *)(data + consumed_bytes);
1352 // ap_log_error(APLOG_MARK, APLOG_DEBUG, APR_SUCCESS, r->server, "CRCSYNC-DECODE inflating %d bytes", strm.avail_in);
1353 // ap_log_hex(APLOG_MARK, APLOG_DEBUG, APR_SUCCESS, r->server, strm.next_in, strm.avail_in);
1354 do {
1355 strm->avail_out = sizeof(decompressed_data_buf);
1356 strm->next_out = decompressed_data_buf;
1357 uInt avail_in_pre_inflate = strm->avail_in;
1358 z_RC = inflate(strm, Z_NO_FLUSH);
1359 if (z_RC == Z_NEED_DICT || z_RC == Z_DATA_ERROR || z_RC == Z_MEM_ERROR)
1360 {
1361 ap_log_error(APLOG_MARK, APLOG_ERR, APR_EGENERAL, r->server, "CRCSYNC-DECODE inflate error: %d", z_RC);
1362 apr_brigade_cleanup(bb);
1363 return APR_EGENERAL;
1364 }
1365 int have = sizeof(decompressed_data_buf) - strm->avail_out;
1366 ap_log_error(APLOG_MARK, APLOG_DEBUG, APR_SUCCESS, r->server,
1367 "CRCSYNC-DECODE inflate rslt %d, consumed %d, produced %d",
1368 z_RC, avail_in_pre_inflate - strm->avail_in, have);
1369 if (have)
1370 {
1371 // write output data
1372 char * buf = apr_palloc(r->pool, have);
1373 memcpy(buf,decompressed_data_buf,have);
1374 apr_bucket * b = apr_bucket_pool_create(buf, have, r->pool, f->c->bucket_alloc);
1375 APR_BRIGADE_INSERT_TAIL(ctx->bb, b);
1376 }
1377 } while (strm->avail_out == 0);
1378 consumed_bytes = len - strm->avail_in;
1379 if (z_RC == Z_STREAM_END)
1380 {
1381 ctx->state = DECODING_NEW_SECTION;
1382 inflateReset(strm);
1383 }
1384 break;
1385 }
1386 default:
1387 {
1388 ap_log_error(APLOG_MARK, APLOG_ERR, APR_SUCCESS, r->server,
1389 "CRCSYNC-DECODE, unknown state %d, terminating transaction",ctx->state);
1390 apr_brigade_cleanup(bb);
1391 return APR_EGENERAL; // TODO: figure out how to pass the error on to the client
1392 }
1393 }
1394 APR_BUCKET_REMOVE(e);
1395 }
1396 }
1397
1398 apr_brigade_cleanup(bb);
1399 return APR_SUCCESS;
1400 }
1401
1402 static void *create_config(apr_pool_t *p, server_rec *s) {
1403 disk_cache_conf *conf = apr_pcalloc(p, sizeof(disk_cache_conf));
1404
1405 /* XXX: Set default values */
1406 conf->dirlevels = DEFAULT_DIRLEVELS;
1407 conf->dirlength = DEFAULT_DIRLENGTH;
1408 conf->maxfs = DEFAULT_MAX_FILE_SIZE;
1409 conf->minfs = DEFAULT_MIN_FILE_SIZE;
1410
1411 conf->cache_root = NULL;
1412 conf->cache_root_len = 0;
1413
1414 return conf;
1415 }
1416
1417 /*
1418 * mod_disk_cache configuration directives handlers.
1419 */
1420 static const char *set_cache_root(cmd_parms *parms, void *in_struct_ptr,
1421 const char *arg) {
1422 disk_cache_conf *conf = ap_get_module_config(parms->server->module_config,
1423 &crccache_client_module);
1424 conf->cache_root = arg;
1425 conf->cache_root_len = strlen(arg);
1426 /* TODO: canonicalize cache_root and strip off any trailing slashes */
1427
1428 return NULL;
1429 }
1430
1431 /*
1432 * Consider eliminating the next two directives in favor of
1433 * Ian's prime number hash...
1434 * key = hash_fn( r->uri)
1435 * filename = "/key % prime1 /key %prime2/key %prime3"
1436 */
1437 static const char *set_cache_dirlevels(cmd_parms *parms, void *in_struct_ptr,
1438 const char *arg) {
1439 disk_cache_conf *conf = ap_get_module_config(parms->server->module_config,
1440 &crccache_client_module);
1441 int val = atoi(arg);
1442 if (val < 1)
1443 return "CacheDirLevelsClient value must be an integer greater than 0";
1444 if (val * conf->dirlength > CACHEFILE_LEN)
1445 return "CacheDirLevelsClient*CacheDirLengthClient value must not be higher than 20";
1446 conf->dirlevels = val;
1447 return NULL;
1448 }
1449 static const char *set_cache_dirlength(cmd_parms *parms, void *in_struct_ptr,
1450 const char *arg) {
1451 disk_cache_conf *conf = ap_get_module_config(parms->server->module_config,
1452 &crccache_client_module);
1453 int val = atoi(arg);
1454 if (val < 1)
1455 return "CacheDirLengthClient value must be an integer greater than 0";
1456 if (val * conf->dirlevels > CACHEFILE_LEN)
1457 return "CacheDirLevelsClient*CacheDirLengthClient value must not be higher than 20";
1458
1459 conf->dirlength = val;
1460 return NULL;
1461 }
1462
1463 static const char *set_cache_minfs(cmd_parms *parms, void *in_struct_ptr,
1464 const char *arg) {
1465 disk_cache_conf *conf = ap_get_module_config(parms->server->module_config,
1466 &crccache_client_module);
1467
1468 if (apr_strtoff(&conf->minfs, arg, NULL, 0) != APR_SUCCESS || conf->minfs
1469 < 0) {
1470 return "CacheMinFileSizeClient argument must be a non-negative integer representing the min size of a file to cache in bytes.";
1471 }
1472 return NULL;
1473 }
1474
1475 static const char *set_cache_maxfs(cmd_parms *parms, void *in_struct_ptr,
1476 const char *arg) {
1477 disk_cache_conf *conf = ap_get_module_config(parms->server->module_config,
1478 &crccache_client_module);
1479 if (apr_strtoff(&conf->maxfs, arg, NULL, 0) != APR_SUCCESS || conf->maxfs
1480 < 0) {
1481 return "CacheMaxFileSizeClient argument must be a non-negative integer representing the max size of a file to cache in bytes.";
1482 }
1483 return NULL;
1484 }
1485
1486 static const command_rec disk_cache_cmds[] = { AP_INIT_TAKE1("CacheRootClient", set_cache_root, NULL, RSRC_CONF,
1487 "The directory to store cache files"), AP_INIT_TAKE1("CacheDirLevelsClient", set_cache_dirlevels, NULL, RSRC_CONF,
1488 "The number of levels of subdirectories in the cache"), AP_INIT_TAKE1("CacheDirLengthClient", set_cache_dirlength, NULL, RSRC_CONF,
1489 "The number of characters in subdirectory names"), AP_INIT_TAKE1("CacheMinFileSizeClient", set_cache_minfs, NULL, RSRC_CONF,
1490 "The minimum file size to cache a document"), AP_INIT_TAKE1("CacheMaxFileSizeClient", set_cache_maxfs, NULL, RSRC_CONF,
1491 "The maximum file size to cache a document"), { NULL } };
1492
1493 static const cache_provider crccache_client_provider = { &remove_entity,
1494 &store_headers, &store_body, &recall_headers, &recall_body,
1495 &create_entity, &open_entity, &remove_url, };
1496
1497 static void disk_cache_register_hook(apr_pool_t *p) {
1498 ap_log_error(APLOG_MARK, APLOG_INFO, 0, NULL,
1499 "Registering crccache client module, (C) 2009, Toby Collett");
1500
1501 /* cache initializer */
1502 ap_register_provider(p, CACHE_PROVIDER_GROUP, "crccache_client", "0",
1503 &crccache_client_provider);
1504 /*
1505 * CACHE_OUT must go into the filter chain after a possible DEFLATE
1506 * filter to ensure that already compressed cache objects do not
1507 * get compressed again. Incrementing filter type by 1 ensures
1508 * his happens.
1509 */
1510 crccache_decode_filter_handle = ap_register_output_filter(
1511 "CRCCACHE_DECODE", crccache_decode_filter, NULL,
1512 AP_FTYPE_CONTENT_SET + 1);
1513 }
1514
1515 module AP_MODULE_DECLARE_DATA crccache_client_module = {
1516 STANDARD20_MODULE_STUFF, NULL, /* create per-directory config structure */
1517 NULL , /* merge per-directory config structures */
1518 create_config, /* create per-server config structure */
1519 NULL , /* merge per-server config structures */
1520 disk_cache_cmds, /* command apr_table_t */
1521 disk_cache_register_hook /* register hooks */
1522 };
Apache HTTP proxy using crcsync