Updated spec
[elliptics.git] / library / meta.c
bloba39aad0051f9a48face3eed25e13b8f43dc9dca8
1 /*
2 * 2008+ Copyright (c) Evgeniy Polyakov <zbr@ioremap.net>
3 * All rights reserved.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
16 #include "config.h"
18 #include <sys/types.h>
19 #include <sys/stat.h>
20 #include <sys/socket.h>
21 #include <sys/mman.h>
22 #include <sys/wait.h>
24 #include <ctype.h>
25 #include <fcntl.h>
26 #include <limits.h>
27 #include <stdio.h>
28 #include <stdlib.h>
29 #include <unistd.h>
31 #include "elliptics.h"
33 #include "elliptics/packet.h"
34 #include "elliptics/interface.h"
36 #define dnet_map_log(n, mask, fmt, a...) do { if ((n)) dnet_log((n), mask, fmt, ##a); else fprintf(stderr, fmt, ##a); } while (0)
38 static int dnet_update_ts_metadata_raw(struct dnet_meta_container *mc, uint64_t flags_set, uint64_t flags_clear)
40 struct dnet_meta m;
41 void *data = mc->data;
42 uint32_t size = mc->size;
43 struct dnet_meta_update *mu;
45 while (size) {
46 if (size < sizeof(struct dnet_meta)) {
47 return -1;
50 m = *(struct dnet_meta *)data;
51 dnet_convert_meta(&m);
53 if (m.size + sizeof(struct dnet_meta) > size) {
54 return -1;
57 if (m.type == DNET_META_UPDATE) {
58 mu = (struct dnet_meta_update *)(data + sizeof(struct dnet_meta));
60 dnet_convert_meta_update(mu);
62 dnet_current_time(&mu->tm);
63 mu->flags |= flags_set;
64 mu->flags &= ~flags_clear;
66 dnet_convert_meta_update(mu);
69 data += m.size + sizeof(struct dnet_meta);
70 size -= m.size + sizeof(struct dnet_meta);
73 return -ENOENT;
76 void dnet_create_meta_update(struct dnet_meta *m, struct timespec *ts, uint64_t flags_set, uint64_t flags_clear)
78 struct dnet_meta_update *mu = (struct dnet_meta_update *)m->data;
80 m->size = sizeof(struct dnet_meta_update);
81 m->type = DNET_META_UPDATE;
83 if (!ts) {
84 dnet_current_time(&mu->tm);
85 } else {
86 mu->tm.tsec = ts->tv_sec;
87 mu->tm.tnsec = ts->tv_nsec;
90 mu->flags = 0;
91 mu->flags |= flags_set;
92 mu->flags &= ~flags_clear;
94 dnet_convert_meta_update(mu);
95 dnet_convert_meta(m);
98 int dnet_update_ts_metadata(struct eblob_backend *b, struct dnet_raw_id *id, uint64_t flags_set, uint64_t flags_clear)
100 int err = 0;
101 struct dnet_meta_container mc;
102 struct dnet_meta *m;
104 memset(&mc, 0, sizeof(struct dnet_meta_container));
106 err = dnet_db_read_raw(b, id, &mc.data);
107 if (err < 0) {
108 m = malloc(sizeof(struct dnet_meta) + sizeof(struct dnet_meta_update));
109 if (!m) {
110 err = -ENOMEM;
111 goto err_out_exit;
113 dnet_create_meta_update(m, NULL, flags_set, flags_clear);
115 mc.data = m;
116 mc.size = sizeof(struct dnet_meta_update) + sizeof(struct dnet_meta);
117 } else {
118 err = dnet_update_ts_metadata_raw(&mc, flags_set, flags_clear);
119 if (err) {
120 /* broken metadata, rewrite it */
121 if (err != -ENOENT) {
122 free(mc.data);
124 mc.data = NULL;
125 mc.size = 0;
128 mc.data = realloc(mc.data, mc.size + sizeof(struct dnet_meta) + sizeof(struct dnet_meta_update));
129 if (!mc.data) {
130 err = -ENOMEM;
131 goto err_out_exit;
134 m = mc.data + mc.size;
135 mc.size += sizeof(struct dnet_meta) + sizeof(struct dnet_meta_update);
137 dnet_create_meta_update(m, NULL, flags_set, flags_clear);
141 err = dnet_db_write_raw(b, id, mc.data, mc.size);
142 if (err) {
143 goto err_out_free;
146 err_out_free:
147 free(mc.data);
148 err_out_exit:
149 return err;
152 struct dnet_meta_update *dnet_get_meta_update(struct dnet_node *n, struct dnet_meta_container *mc,
153 struct dnet_meta_update *meta_update)
155 struct dnet_meta m;
156 void *data = mc->data;
157 uint32_t size = mc->size;
158 struct dnet_meta_update *mu;
159 int num;
161 while (size) {
162 if (size < sizeof(struct dnet_meta)) {
163 dnet_log(n, DNET_LOG_ERROR, "Metadata size %u is too small, min %zu, searching for type 0x%x.\n",
164 size, sizeof(struct dnet_meta), DNET_META_UPDATE);
165 return NULL;
168 m = *(struct dnet_meta *)data;
169 dnet_convert_meta(&m);
171 if (m.size + sizeof(struct dnet_meta) > size) {
172 dnet_log(n, DNET_LOG_ERROR, "Metadata entry broken: entry size %u, type: 0x%x, struct size: %zu, "
173 "total size left: %u, searching for type: 0x%x.\n",
174 m.size, m.type, sizeof(struct dnet_meta), size, DNET_META_UPDATE);
175 return NULL;
178 if (m.type == DNET_META_UPDATE) {
179 mu = (struct dnet_meta_update *)(data + sizeof(struct dnet_meta));
181 num = m.size / sizeof(struct dnet_meta_update);
182 if (num >= 0) {
183 if (meta_update) {
184 memcpy(meta_update, &mu[0], sizeof(struct dnet_meta_update));
185 dnet_convert_meta_update(meta_update);
187 return &mu[0];
191 data += m.size + sizeof(struct dnet_meta);
192 size -= m.size + sizeof(struct dnet_meta);
194 return NULL;
197 struct dnet_meta *dnet_meta_search(struct dnet_node *n, struct dnet_meta_container *mc, uint32_t type)
199 void *data = mc->data;
200 uint32_t size = mc->size;
201 struct dnet_meta m, *found = NULL;
203 while (size) {
204 if (size < sizeof(struct dnet_meta)) {
205 dnet_map_log(n, DNET_LOG_ERROR, "Metadata size %u is too small, min %zu, searching for type 0x%x.\n",
206 size, sizeof(struct dnet_meta), type);
207 break;
210 m = *(struct dnet_meta *)data;
211 dnet_convert_meta(&m);
213 if (m.size + sizeof(struct dnet_meta) > size) {
214 dnet_map_log(n, DNET_LOG_ERROR, "Metadata entry broken: entry size %u, type: 0x%x, struct size: %zu, "
215 "total size left: %u, searching for type: 0x%x.\n",
216 m.size, m.type, sizeof(struct dnet_meta), size, type);
217 break;
220 if (m.type == type) {
221 found = data;
222 break;
225 data += m.size + sizeof(struct dnet_meta);
226 size -= m.size + sizeof(struct dnet_meta);
229 return found;
232 void dnet_convert_metadata(struct dnet_node *n __unused, void *data, int size)
234 void *ptr = data;
235 struct dnet_meta *m;
237 while (size) {
238 m = ptr;
240 ptr += sizeof(struct dnet_meta) + m->size;
241 size -= sizeof(struct dnet_meta) + m->size;
243 dnet_convert_meta(m);
247 int dnet_write_metadata(struct dnet_node *n, struct dnet_meta_container *mc, int convert, uint64_t cflags)
249 struct dnet_io_control ctl;
250 void *result;
251 int err;
253 if (n->flags & DNET_CFG_NO_META)
254 return 0;
256 if (convert)
257 dnet_convert_metadata(n, mc->data, mc->size);
259 memset(&ctl, 0, sizeof(ctl));
261 ctl.fd = -1;
263 ctl.data = mc->data;
264 ctl.io.size = mc->size;
265 ctl.io.flags = DNET_IO_FLAGS_META;
266 ctl.cflags = cflags;
268 memcpy(&ctl.id, &mc->id, sizeof(struct dnet_id));
269 ctl.id.type = ctl.io.type = EBLOB_TYPE_META;
271 err = dnet_write_data_wait(n, &ctl, &result);
272 if (err < 0)
273 return err;
275 free(result);
277 return 0;
280 int dnet_create_write_metadata_strings(struct dnet_node *n, const void *remote, unsigned int remote_len,
281 struct dnet_id *id, struct timespec *ts, uint64_t cflags)
283 struct dnet_metadata_control mc;
284 int *groups = NULL;
285 int group_num = 0;
286 int err;
288 pthread_mutex_lock(&n->group_lock);
289 group_num = n->group_num;
290 groups = alloca(group_num * sizeof(int));
292 memcpy(groups, n->groups, group_num * sizeof(int));
293 pthread_mutex_unlock(&n->group_lock);
295 memset(&mc, 0, sizeof(mc));
296 mc.obj = remote;
297 mc.len = remote_len;
298 mc.groups = groups;
299 mc.group_num = group_num;
300 mc.id = *id;
301 mc.cflags = cflags;
303 if (ts) {
304 mc.ts = *ts;
305 } else {
306 struct timeval tv;
308 gettimeofday(&tv, NULL);
309 mc.ts.tv_sec = tv.tv_sec;
310 mc.ts.tv_nsec = tv.tv_usec * 1000;
313 err = dnet_create_write_metadata(n, &mc);
314 if (err < 0) {
315 dnet_log(n, DNET_LOG_ERROR, "%s: failed to write metadata: %d\n", dnet_dump_id(id), err);
318 return 0;
321 int dnet_create_metadata(struct dnet_node *n, struct dnet_metadata_control *ctl, struct dnet_meta_container *mc)
323 struct dnet_meta_check_status *c;
324 struct dnet_meta *m;
325 int size = 0, err, nsize = 0;
326 void *ns;
328 size += sizeof(struct dnet_meta_check_status) + sizeof(struct dnet_meta);
330 if (ctl->obj && ctl->len)
331 size += ctl->len + sizeof(struct dnet_meta);
333 if (ctl->groups && ctl->group_num)
334 size += ctl->group_num * sizeof(int) + sizeof(struct dnet_meta);
336 size += sizeof(struct dnet_meta_update) + sizeof(struct dnet_meta);
338 ns = dnet_node_get_ns(n, &nsize);
339 if (ns && nsize)
340 size += nsize + sizeof(struct dnet_meta);
342 if (!size) {
343 err = -EINVAL;
344 goto err_out_exit;
347 mc->data = malloc(size);
348 if (!mc->data) {
349 err = -ENOMEM;
350 goto err_out_exit;
352 memset(mc->data, 0, size);
354 m = (struct dnet_meta *)(mc->data);
356 c = (struct dnet_meta_check_status *)m->data;
357 m->size = sizeof(struct dnet_meta_check_status);
358 m->type = DNET_META_CHECK_STATUS;
360 /* Check status is undefined for now, it will be filled during actual check */
361 memset(c, 0, sizeof(struct dnet_meta_check_status));
363 m = (struct dnet_meta *)(m->data + m->size);
364 dnet_create_meta_update(m, ctl->ts.tv_sec ? &ctl->ts : NULL, 0, 0);
366 m = (struct dnet_meta *)(m->data + m->size);
368 if (ctl->obj && ctl->len) {
369 m->size = ctl->len;
370 m->type = DNET_META_PARENT_OBJECT;
371 memcpy(m->data, ctl->obj, ctl->len);
373 m = (struct dnet_meta *)(m->data + m->size);
376 if (ctl->groups && ctl->group_num) {
377 m->size = ctl->group_num * sizeof(int);
378 m->type = DNET_META_GROUPS;
379 memcpy(m->data, ctl->groups, ctl->group_num * sizeof(int));
381 m = (struct dnet_meta *)(m->data + m->size);
384 if (ns && nsize) {
385 m->size = nsize;
386 m->type = DNET_META_NAMESPACE;
387 memcpy(m->data, ns, nsize);
389 m = (struct dnet_meta *)(m->data + m->size);
392 mc->size = size;
393 memcpy(&mc->id, &ctl->id, sizeof(struct dnet_id));
394 err = 0;
396 err_out_exit:
397 return err;
400 int dnet_create_write_metadata(struct dnet_node *n, struct dnet_metadata_control *ctl)
402 struct dnet_meta_container mc;
403 int err;
405 memset(&mc, 0, sizeof(struct dnet_meta_container));
407 err = dnet_create_metadata(n, ctl, &mc);
408 if (err)
409 goto err_out_exit;
411 err = dnet_write_metadata(n, &mc, 1, ctl->cflags);
412 if (err)
413 goto err_out_free;
415 err_out_free:
416 free(mc.data);
417 err_out_exit:
418 return err;
421 static char *dnet_meta_types[__DNET_META_MAX] = {
422 [DNET_META_PARENT_OBJECT] = "DNET_META_PARENT_OBJECT",
423 [DNET_META_GROUPS] = "DNET_META_GROUPS",
424 [DNET_META_CHECK_STATUS] = "DNET_META_CHECK_STATUS",
425 [DNET_META_NAMESPACE] = "DNET_META_NAMESPACE",
426 [DNET_META_UPDATE] = "DNET_META_UPDATE",
427 [DNET_META_CHECKSUM] = "DNET_META_CHECKSUM",
430 void dnet_meta_print(struct dnet_node *n, struct dnet_meta_container *mc)
432 void *data;
433 int size, err;
434 struct dnet_meta *m;
435 char tstr[64];
436 struct tm tm;
438 data = mc->data;
439 size = mc->size;
441 dnet_log(n, DNET_LOG_DATA, "%s: size: %u\n", dnet_dump_id_len(&mc->id, DNET_ID_SIZE), mc->size);
443 while (size) {
444 m = data;
446 dnet_convert_meta(m);
448 if (m->type >= __DNET_META_MAX || m->type < DNET_META_PARENT_OBJECT) {
449 dnet_log(n, DNET_LOG_ERROR, "%s: incorrect meta type %d\n", dnet_dump_id(&mc->id), m->type);
450 return;
453 if (m->type == DNET_META_PARENT_OBJECT) {
454 char name[m->size + 1];
456 memcpy(name, m->data, m->size);
457 name[m->size] = '\0';
458 dnet_log(n, DNET_LOG_DATA, "%s: type: %u, size: %u, name: '%s'\n",
459 dnet_meta_types[m->type], m->type, m->size, name);
460 } else if (m->type == DNET_META_GROUPS) {
461 int *groups = (int *)m->data;
462 int gnum = m->size / sizeof(int);
463 char str[gnum * 36 + 1], *ptr;
464 int i, rest;
466 memset(str, 0, sizeof(str));
468 ptr = str;
469 rest = sizeof(str);
470 for (i=0; i<gnum; ++i) {
471 err = snprintf(ptr, rest, "%d:", groups[i]);
472 if (err > rest)
473 break;
475 rest -= err;
476 ptr += err;
478 if (i == gnum - 1)
479 *(--ptr) = '\0';
482 dnet_log(n, DNET_LOG_DATA, "%s: type: %u, size: %u, groups: %s\n",
483 dnet_meta_types[m->type], m->type, m->size, str);
484 } else if (m->type == DNET_META_CHECK_STATUS) {
485 struct dnet_meta_check_status *s = (struct dnet_meta_check_status *)m->data;
487 dnet_convert_meta_check_status(s);
489 localtime_r((time_t *)&s->tm.tsec, &tm);
490 strftime(tstr, sizeof(tstr), "%F %R:%S %Z", &tm);
492 dnet_log(n, DNET_LOG_DATA, "%s: type: %u, size: %u, check status: %d, ts: %s: %lld.%lld\n",
493 dnet_meta_types[m->type],
494 m->type, m->size, s->status, tstr,
495 (unsigned long long)s->tm.tsec,
496 (unsigned long long)s->tm.tnsec);
497 } else if (m->type == DNET_META_UPDATE) {
498 struct dnet_meta_update *mu = (struct dnet_meta_update *)m->data;
500 dnet_convert_meta_update(mu);
502 localtime_r((time_t *)&mu->tm.tsec, &tm);
503 strftime(tstr, sizeof(tstr), "%F %R:%S %Z", &tm);
505 dnet_log(n, DNET_LOG_DATA, "%s: type: %u, size: %u, flags: %llx, ts: %s %lld.%lld\n",
506 dnet_meta_types[m->type], m->type, m->size,
507 (unsigned long long)mu->flags, tstr,
508 (unsigned long long)mu->tm.tsec, (unsigned long long)mu->tm.tnsec);
509 } else if (m->type == DNET_META_NAMESPACE) {
510 char str[m->size + 1];
511 memcpy(str, m->data, m->size);
512 str[m->size] = '\0';
514 dnet_log(n, DNET_LOG_DATA, "%s: type: %u, size: %u, namespace: %s\n",
515 dnet_meta_types[m->type], m->type, m->size, str);
516 } else if (m->type == DNET_META_CHECKSUM) {
517 struct dnet_meta_checksum *cs = (struct dnet_meta_checksum *)m->data;
518 char str[2*DNET_CSUM_SIZE+1];
520 localtime_r((time_t *)&cs->tm.tsec, &tm);
521 strftime(tstr, sizeof(tstr), "%F %R:%S %Z", &tm);
523 dnet_dump_id_len_raw(cs->checksum, DNET_CSUM_SIZE, str);
524 dnet_log(n, DNET_LOG_DATA, "%s: type: %u, size: %u, csum: %s, ts: %s %lld.%lld\n",
525 dnet_meta_types[m->type], m->type, m->size, str, tstr,
526 (unsigned long long)cs->tm.tsec, (unsigned long long)cs->tm.tnsec);
527 } else {
528 dnet_log(n, DNET_LOG_DATA, "%s: type: %u, size: %u\n",
529 dnet_meta_types[m->type], m->type, m->size);
532 data += m->size + sizeof(*m);
533 size -= m->size + sizeof(*m);
537 int dnet_read_meta(struct dnet_node *n, struct dnet_meta_container *mc,
538 const void *remote, unsigned int remote_len, struct dnet_id *id)
540 struct dnet_io_attr io;
541 struct dnet_id raw;
542 void *data;
543 int err;
545 io.flags = DNET_IO_FLAGS_META;
546 io.size = 0;
547 io.offset = 0;
548 io.type = 0;
549 io.start = io.num = 0;
551 if (!id) {
552 if (!remote) {
553 err = -EINVAL;
554 goto err_out_exit;
557 dnet_transform(n, remote, remote_len, &raw);
558 id = &raw;
560 id->type = 0;
562 memcpy(io.id, id->id, DNET_ID_SIZE);
563 memcpy(io.parent, id->id, DNET_ID_SIZE);
565 data = dnet_read_data_wait(n, id, &io, 0, &err);
566 } else {
567 id->type = 0;
569 memcpy(io.id, id->id, DNET_ID_SIZE);
570 memcpy(io.parent, id->id, DNET_ID_SIZE);
572 data = dnet_read_data_wait_raw(n, id, &io, DNET_CMD_READ, 0, &err);
575 if (data) {
576 io.size -= sizeof(struct dnet_io_attr);
578 mc->data = malloc(io.size);
579 if (!mc->data) {
580 err = -ENOMEM;
581 goto err_out_free;
584 memcpy(mc->data, data + sizeof(struct dnet_io_attr), io.size);
586 memcpy(&mc->id, id, sizeof(struct dnet_id));
587 mc->size = io.size;
590 err_out_free:
591 free(data);
592 err_out_exit:
593 return err;
596 int dnet_meta_update_check_status_raw(struct dnet_node *n, struct dnet_meta_container *mc)
598 struct dnet_meta *m = NULL;
599 struct dnet_meta_check_status *meta_check;
600 int err = 0;
602 m = dnet_meta_search(n, mc, DNET_META_CHECK_STATUS);
604 if (!m) {
605 mc->data = realloc(mc->data, mc->size + sizeof(struct dnet_meta) + sizeof(struct dnet_meta_check_status));
606 if (!mc->data) {
607 err = -ENOMEM;
608 goto err_out_free;
611 m = mc->data + mc->size;
612 mc->size += sizeof(struct dnet_meta) + sizeof(struct dnet_meta_check_status);
614 m->type = DNET_META_CHECK_STATUS;
615 m->size = sizeof(struct dnet_meta_check_status);
616 dnet_convert_meta(m);
619 meta_check = (struct dnet_meta_check_status *)m->data;
620 meta_check->status = 0;
622 dnet_current_time(&meta_check->tm);
624 dnet_convert_meta_check_status(meta_check);
626 return err;
628 err_out_free:
629 free(mc->data);
630 return err;
633 int dnet_meta_update_check_status(struct dnet_node *n, struct dnet_meta_container *mc)
635 struct dnet_raw_id id;
636 int err;
638 err = dnet_meta_update_check_status_raw(n, mc);
640 if (err) {
641 dnet_log(n, DNET_LOG_ERROR, "%s: failed to update DNET_META_CHECK_STATUS, err=%d\n",
642 dnet_dump_id(&mc->id), err);
643 return err;
644 } else {
645 memcpy(&id.id, &mc->id.id, DNET_ID_SIZE);
646 err = n->cb->meta_write(n->cb->command_private, &id, mc->data, mc->size);
647 if (err) {
648 dnet_log(n, DNET_LOG_ERROR, "%s: failed to write meta, err=%d\n",
649 dnet_dump_id(&mc->id), err);
653 return err;