library/meta.c

   1 /*
   2  * 2008+ Copyright (c) Evgeniy Polyakov <zbr@ioremap.net>
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  */
  15
  16 #include <sys/types.h>
  17 #include <sys/stat.h>
  18 #include <sys/socket.h>
  19 #include <sys/mman.h>
  20 #include <sys/wait.h>
  21
  22 #include <ctype.h>
  23 #include <fcntl.h>
  24 #include <limits.h>
  25 #include <stdio.h>
  26 #include <stdlib.h>
  27 #include <unistd.h>
  28
  29 #include "elliptics.h"
  30
  31 #include "elliptics/packet.h"
  32 #include "elliptics/interface.h"
  33
  34 #define dnet_map_log(n, level, fmt, a...) do { if ((n)) dnet_log((n), level, fmt, ##a); else fprintf(stderr, fmt, ##a); } while (0)
  35
  36 int dnet_update_ts_metadata_raw(struct dnet_meta_container *mc, uint64_t flags_set, uint64_t flags_clear)
  37 {
  38         struct dnet_meta m;
  39         void *data = mc->data;
  40         uint32_t size = mc->size;
  41         struct dnet_meta_update *mu;
  42
  43         while (size) {
  44                 if (size < sizeof(struct dnet_meta)) {
  45                         return -1;
  46                 }
  47
  48                 m = *(struct dnet_meta *)data;
  49                 dnet_convert_meta(&m);
  50
  51                 if (m.size + sizeof(struct dnet_meta) > size) {
  52                         return -1;
  53                 }
  54
  55                 if (m.type == DNET_META_UPDATE) {
  56                         mu = (struct dnet_meta_update *)(data + sizeof(struct dnet_meta));
  57
  58                         dnet_convert_meta_update(mu);
  59
  60                         dnet_current_time(&mu->tm);
  61                         mu->flags |= flags_set;
  62                         mu->flags &= ~flags_clear;
  63
  64                         dnet_convert_meta_update(mu);
  65                 }
  66
  67                 data += m.size + sizeof(struct dnet_meta);
  68                 size -= m.size + sizeof(struct dnet_meta);
  69         }
  70
  71         return -ENOENT;
  72 }
  73
  74 void dnet_create_meta_update(struct dnet_meta *m, struct timespec *ts, uint64_t flags_set, uint64_t flags_clear)
  75 {
  76         struct dnet_meta_update *mu = (struct dnet_meta_update *)m->data;
  77
  78         m->size = sizeof(struct dnet_meta_update);
  79         m->type = DNET_META_UPDATE;
  80
  81         if (!ts) {
  82                 dnet_current_time(&mu->tm);
  83         } else {
  84                 mu->tm.tsec = ts->tv_sec;
  85                 mu->tm.tnsec = ts->tv_nsec;
  86         }
  87
  88         mu->flags = 0;
  89         mu->flags |= flags_set;
  90         mu->flags &= ~flags_clear;
  91
  92         dnet_convert_meta_update(mu);
  93         dnet_convert_meta(m);
  94 }
  95
  96 struct dnet_meta_update *dnet_get_meta_update(struct dnet_node *n, struct dnet_meta_container *mc,
  97                 struct dnet_meta_update *meta_update)
  98 {
  99         struct dnet_meta m;
 100         void *data = mc->data;
 101         uint32_t size = mc->size;
 102         struct dnet_meta_update *mu;
 103         int num;
 104
 105         while (size) {
 106                 if (size < sizeof(struct dnet_meta)) {
 107                         dnet_log(n, DNET_LOG_ERROR, "Metadata size %u is too small, min %zu, searching for type 0x%x.\n",
 108                                         size, sizeof(struct dnet_meta), DNET_META_UPDATE);
 109                         return NULL;
 110                 }
 111
 112                 m = *(struct dnet_meta *)data;
 113                 dnet_convert_meta(&m);
 114
 115                 if (m.size + sizeof(struct dnet_meta) > size) {
 116                         dnet_log(n, DNET_LOG_ERROR, "Metadata entry broken: entry size %u, type: 0x%x, struct size: %zu, "
 117                                         "total size left: %u, searching for type: 0x%x.\n",
 118                                         m.size, m.type, sizeof(struct dnet_meta), size, DNET_META_UPDATE);
 119                         return NULL;
 120                 }
 121
 122                 if (m.type == DNET_META_UPDATE) {
 123                         mu = (struct dnet_meta_update *)(data + sizeof(struct dnet_meta));
 124
 125                         num = m.size / sizeof(struct dnet_meta_update);
 126                         if (num >= 0) {
 127                                 if (meta_update) {
 128                                         memcpy(meta_update, &mu[0], sizeof(struct dnet_meta_update));
 129                                         dnet_convert_meta_update(meta_update);
 130                                 }
 131                                 return &mu[0];
 132                         }
 133                 }
 134
 135                 data += m.size + sizeof(struct dnet_meta);
 136                 size -= m.size + sizeof(struct dnet_meta);
 137         }
 138         return NULL;
 139 }
 140
 141 struct dnet_meta *dnet_meta_search(struct dnet_node *n, struct dnet_meta_container *mc, uint32_t type)
 142 {
 143         void *data = mc->data;
 144         uint32_t size = mc->size;
 145         struct dnet_meta m, *found = NULL;
 146
 147         while (size) {
 148                 if (size < sizeof(struct dnet_meta)) {
 149                         dnet_map_log(n, DNET_LOG_ERROR, "Metadata size %u is too small, min %zu, searching for type 0x%x.\n",
 150                                         size, sizeof(struct dnet_meta), type);
 151                         break;
 152                 }
 153
 154                 m = *(struct dnet_meta *)data;
 155                 dnet_convert_meta(&m);
 156
 157                 if (m.size + sizeof(struct dnet_meta) > size) {
 158                         dnet_map_log(n, DNET_LOG_ERROR, "Metadata entry broken: entry size %u, type: 0x%x, struct size: %zu, "
 159                                         "total size left: %u, searching for type: 0x%x.\n",
 160                                         m.size, m.type, sizeof(struct dnet_meta), size, type);
 161                         break;
 162                 }
 163
 164                 if (m.type == type) {
 165                         found = data;
 166                         break;
 167                 }
 168
 169                 data += m.size + sizeof(struct dnet_meta);
 170                 size -= m.size + sizeof(struct dnet_meta);
 171         }
 172
 173         return found;
 174 }
 175
 176 void dnet_convert_metadata(struct dnet_node *n __unused, void *data, int size)
 177 {
 178         void *ptr = data;
 179         struct dnet_meta *m;
 180
 181         while (size) {
 182                 m = ptr;
 183
 184                 ptr += sizeof(struct dnet_meta) + m->size;
 185                 size -= sizeof(struct dnet_meta) + m->size;
 186
 187                 dnet_convert_meta(m);
 188         }
 189 }
 190
 191 int dnet_write_metadata(struct dnet_node *n, struct dnet_meta_container *mc, int convert, uint64_t cflags)
 192 {
 193         struct dnet_io_control ctl;
 194         void *result;
 195         int err;
 196
 197         if (n->flags & DNET_CFG_NO_META)
 198                 return 0;
 199
 200         if (convert)
 201                 dnet_convert_metadata(n, mc->data, mc->size);
 202
 203         memset(&ctl, 0, sizeof(ctl));
 204
 205         ctl.fd = -1;
 206
 207         ctl.data = mc->data;
 208         ctl.io.size = mc->size;
 209         ctl.io.flags = DNET_IO_FLAGS_META;
 210         ctl.cflags = cflags;
 211
 212         memcpy(&ctl.id, &mc->id, sizeof(struct dnet_id));
 213         ctl.id.type = ctl.io.type = EBLOB_TYPE_META;
 214
 215         err = dnet_write_data_wait(n, &ctl, &result);
 216         if (err < 0)
 217                 return err;
 218
 219         free(result);
 220
 221         return 0;
 222 }
 223
 224 int dnet_create_write_metadata_strings(struct dnet_node *n, const void *remote, unsigned int remote_len,
 225                 struct dnet_id *id, struct timespec *ts, uint64_t cflags)
 226 {
 227         struct dnet_metadata_control mc;
 228         int *groups = NULL;
 229         int group_num = 0;
 230         int err;
 231
 232         pthread_mutex_lock(&n->group_lock);
 233         group_num = n->group_num;
 234         groups = alloca(group_num * sizeof(int));
 235
 236         memcpy(groups, n->groups, group_num * sizeof(int));
 237         pthread_mutex_unlock(&n->group_lock);
 238
 239         memset(&mc, 0, sizeof(mc));
 240         mc.obj = remote;
 241         mc.len = remote_len;
 242         mc.groups = groups;
 243         mc.group_num = group_num;
 244         mc.id = *id;
 245         mc.cflags = cflags;
 246
 247         if (ts) {
 248                 mc.ts = *ts;
 249         } else {
 250                 struct timeval tv;
 251
 252                 gettimeofday(&tv, NULL);
 253                 mc.ts.tv_sec = tv.tv_sec;
 254                 mc.ts.tv_nsec = tv.tv_usec * 1000;
 255         }
 256
 257         err = dnet_create_write_metadata(n, &mc);
 258         if (err < 0) {
 259                 dnet_log(n, DNET_LOG_ERROR, "%s: failed to write metadata: %d\n", dnet_dump_id(id), err);
 260         }
 261
 262         return 0;
 263 }
 264
 265 int dnet_create_metadata(struct dnet_node *n, struct dnet_metadata_control *ctl, struct dnet_meta_container *mc)
 266 {
 267         struct dnet_meta_check_status *c;
 268         struct dnet_meta *m;
 269         int size = 0, err, nsize = 0;
 270         void *ns;
 271
 272         size += sizeof(struct dnet_meta_check_status) + sizeof(struct dnet_meta);
 273
 274         if (ctl->obj && ctl->len)
 275                 size += ctl->len + sizeof(struct dnet_meta);
 276
 277         if (ctl->groups && ctl->group_num)
 278                 size += ctl->group_num * sizeof(int) + sizeof(struct dnet_meta);
 279
 280         size += sizeof(struct dnet_meta_update) + sizeof(struct dnet_meta);
 281
 282         ns = dnet_node_get_ns(n, &nsize);
 283         if (ns && nsize)
 284                 size += nsize + sizeof(struct dnet_meta);
 285
 286         if (!size) {
 287                 err = -EINVAL;
 288                 goto err_out_exit;
 289         }
 290
 291         mc->data = malloc(size);
 292         if (!mc->data) {
 293                 err = -ENOMEM;
 294                 goto err_out_exit;
 295         }
 296         memset(mc->data, 0, size);
 297
 298         m = (struct dnet_meta *)(mc->data);
 299
 300         c = (struct dnet_meta_check_status *)m->data;
 301         m->size = sizeof(struct dnet_meta_check_status);
 302         m->type = DNET_META_CHECK_STATUS;
 303
 304         /* Check status is undefined for now, it will be filled during actual check */
 305         memset(c, 0, sizeof(struct dnet_meta_check_status));
 306
 307         m = (struct dnet_meta *)(m->data + m->size);
 308         dnet_create_meta_update(m, ctl->ts.tv_sec ? &ctl->ts : NULL, 0, 0);
 309
 310         m = (struct dnet_meta *)(m->data + m->size);
 311
 312         if (ctl->obj && ctl->len) {
 313                 m->size = ctl->len;
 314                 m->type = DNET_META_PARENT_OBJECT;
 315                 memcpy(m->data, ctl->obj, ctl->len);
 316
 317                 m = (struct dnet_meta *)(m->data + m->size);
 318         }
 319
 320         if (ctl->groups && ctl->group_num) {
 321                 m->size = ctl->group_num * sizeof(int);
 322                 m->type = DNET_META_GROUPS;
 323                 memcpy(m->data, ctl->groups, ctl->group_num * sizeof(int));
 324
 325                 m = (struct dnet_meta *)(m->data + m->size);
 326         }
 327
 328         if (ns && nsize) {
 329                 m->size = nsize;
 330                 m->type = DNET_META_NAMESPACE;
 331                 memcpy(m->data, ns, nsize);
 332
 333                 m = (struct dnet_meta *)(m->data + m->size);
 334         }
 335
 336         mc->size = size;
 337         memcpy(&mc->id, &ctl->id, sizeof(struct dnet_id));
 338         err = 0;
 339
 340 err_out_exit:
 341         return err;
 342 }
 343
 344 int dnet_create_write_metadata(struct dnet_node *n, struct dnet_metadata_control *ctl)
 345 {
 346         struct dnet_meta_container mc;
 347         int err;
 348
 349         memset(&mc, 0, sizeof(struct dnet_meta_container));
 350
 351         err = dnet_create_metadata(n, ctl, &mc);
 352         if (err)
 353                 goto err_out_exit;
 354
 355         err = dnet_write_metadata(n, &mc, 1, ctl->cflags);
 356         if (err)
 357                 goto err_out_free;
 358
 359 err_out_free:
 360         free(mc.data);
 361 err_out_exit:
 362         return err;
 363 }
 364
 365 static char *dnet_meta_types[__DNET_META_MAX] = {
 366         [DNET_META_PARENT_OBJECT] = "DNET_META_PARENT_OBJECT",
 367         [DNET_META_GROUPS] = "DNET_META_GROUPS",
 368         [DNET_META_CHECK_STATUS] = "DNET_META_CHECK_STATUS",
 369         [DNET_META_NAMESPACE] = "DNET_META_NAMESPACE",
 370         [DNET_META_UPDATE] = "DNET_META_UPDATE",
 371         [DNET_META_CHECKSUM] = "DNET_META_CHECKSUM",
 372 };
 373
 374 void dnet_meta_print(struct dnet_node *n, struct dnet_meta_container *mc)
 375 {
 376         void *data;
 377         int size, err;
 378         struct dnet_meta *m;
 379         char tstr[64];
 380         struct tm tm;
 381
 382         data = mc->data;
 383         size = mc->size;
 384
 385         dnet_log(n, DNET_LOG_DATA, "%s: size: %u\n", dnet_dump_id_len(&mc->id, DNET_ID_SIZE), mc->size);
 386
 387         while (size) {
 388                 m = data;
 389
 390                 dnet_convert_meta(m);
 391
 392                 if (m->type >= __DNET_META_MAX || m->type < DNET_META_PARENT_OBJECT) {
 393                         dnet_log(n, DNET_LOG_ERROR, "%s: incorrect meta type %d\n", dnet_dump_id(&mc->id), m->type);
 394                         return;
 395                 }
 396
 397                 if (m->type == DNET_META_PARENT_OBJECT) {
 398                         char name[m->size + 1];
 399
 400                         memcpy(name, m->data, m->size);
 401                         name[m->size] = '\0';
 402                         dnet_log(n, DNET_LOG_DATA, "%s: type: %u, size: %u, name: '%s'\n",
 403                                         dnet_meta_types[m->type], m->type, m->size, name);
 404                 } else if (m->type == DNET_META_GROUPS) {
 405                         int *groups = (int *)m->data;
 406                         int gnum = m->size / sizeof(int);
 407                         char str[gnum * 36 + 1], *ptr;
 408                         int i, rest;
 409
 410                         memset(str, 0, sizeof(str));
 411
 412                         ptr = str;
 413                         rest = sizeof(str);
 414                         for (i=0; i<gnum; ++i) {
 415                                 err = snprintf(ptr, rest, "%d:", groups[i]);
 416                                 if (err > rest)
 417                                         break;
 418
 419                                 rest -= err;
 420                                 ptr += err;
 421
 422                                 if (i == gnum - 1)
 423                                         *(--ptr) = '\0';
 424                         }
 425
 426                         dnet_log(n, DNET_LOG_DATA, "%s: type: %u, size: %u, groups: %s\n",
 427                                         dnet_meta_types[m->type], m->type, m->size, str);
 428                 } else if (m->type == DNET_META_CHECK_STATUS) {
 429                         struct dnet_meta_check_status *s = (struct dnet_meta_check_status *)m->data;
 430
 431                         dnet_convert_meta_check_status(s);
 432
 433                         localtime_r((time_t *)&s->tm.tsec, &tm);
 434                         strftime(tstr, sizeof(tstr), "%F %R:%S %Z", &tm);
 435
 436                         dnet_log(n, DNET_LOG_DATA, "%s: type: %u, size: %u, check status: %d, ts: %s: %lld.%lld\n",
 437                                         dnet_meta_types[m->type],
 438                                         m->type, m->size, s->status, tstr,
 439                                         (unsigned long long)s->tm.tsec,
 440                                         (unsigned long long)s->tm.tnsec);
 441                 } else if (m->type == DNET_META_UPDATE) {
 442                         struct dnet_meta_update *mu = (struct dnet_meta_update *)m->data;
 443
 444                         dnet_convert_meta_update(mu);
 445
 446                         localtime_r((time_t *)&mu->tm.tsec, &tm);
 447                         strftime(tstr, sizeof(tstr), "%F %R:%S %Z", &tm);
 448
 449                         dnet_log(n, DNET_LOG_DATA, "%s: type: %u, size: %u, flags: %llx, ts: %s %lld.%lld\n",
 450                                         dnet_meta_types[m->type], m->type, m->size,
 451                                         (unsigned long long)mu->flags, tstr,
 452                                         (unsigned long long)mu->tm.tsec, (unsigned long long)mu->tm.tnsec);
 453                 } else if (m->type == DNET_META_NAMESPACE) {
 454                         char str[m->size + 1];
 455                         memcpy(str, m->data, m->size);
 456                         str[m->size] = '\0';
 457
 458                         dnet_log(n, DNET_LOG_DATA, "%s: type: %u, size: %u, namespace: %s\n",
 459                                         dnet_meta_types[m->type], m->type, m->size, str);
 460                 } else if (m->type == DNET_META_CHECKSUM) {
 461                         struct dnet_meta_checksum *cs = (struct dnet_meta_checksum *)m->data;
 462                         char str[2*DNET_CSUM_SIZE+1];
 463
 464                         localtime_r((time_t *)&cs->tm.tsec, &tm);
 465                         strftime(tstr, sizeof(tstr), "%F %R:%S %Z", &tm);
 466
 467                         dnet_dump_id_len_raw(cs->checksum, DNET_CSUM_SIZE, str);
 468                         dnet_log(n, DNET_LOG_DATA, "%s: type: %u, size: %u, csum: %s, ts: %s %lld.%lld\n",
 469                                         dnet_meta_types[m->type], m->type, m->size, str, tstr,
 470                                         (unsigned long long)cs->tm.tsec, (unsigned long long)cs->tm.tnsec);
 471                 } else {
 472                         dnet_log(n, DNET_LOG_DATA, "%s: type: %u, size: %u\n",
 473                                         dnet_meta_types[m->type], m->type, m->size);
 474                 }
 475
 476                 data += m->size + sizeof(*m);
 477                 size -= m->size + sizeof(*m);
 478         }
 479 }
 480
 481 int dnet_read_meta(struct dnet_node *n, struct dnet_meta_container *mc,
 482                 const void *remote, unsigned int remote_len, struct dnet_id *id)
 483 {
 484         struct dnet_io_attr io;
 485         struct dnet_id raw;
 486         void *data;
 487         int err;
 488
 489         io.flags = DNET_IO_FLAGS_META;
 490         io.size = 0;
 491         io.offset = 0;
 492         io.type = 0;
 493         io.start = io.num = 0;
 494
 495         if (!id) {
 496                 if (!remote) {
 497                         err = -EINVAL;
 498                         goto err_out_exit;
 499                 }
 500
 501                 dnet_transform(n, remote, remote_len, &raw);
 502                 id = &raw;
 503
 504                 id->type = 0;
 505
 506                 memcpy(io.id, id->id, DNET_ID_SIZE);
 507                 memcpy(io.parent, id->id, DNET_ID_SIZE);
 508
 509                 data = dnet_read_data_wait(n, id, &io, 0, &err);
 510         } else {
 511                 id->type = 0;
 512
 513                 memcpy(io.id, id->id, DNET_ID_SIZE);
 514                 memcpy(io.parent, id->id, DNET_ID_SIZE);
 515
 516                 data = dnet_read_data_wait_raw(n, id, &io, DNET_CMD_READ, 0, &err);
 517         }
 518
 519         if (data) {
 520                 io.size -= sizeof(struct dnet_io_attr);
 521
 522                 mc->data = malloc(io.size);
 523                 if (!mc->data) {
 524                         err = -ENOMEM;
 525                         goto err_out_free;
 526                 }
 527
 528                 memcpy(mc->data, data + sizeof(struct dnet_io_attr), io.size);
 529
 530                 memcpy(&mc->id, id, sizeof(struct dnet_id));
 531                 mc->size = io.size;
 532         }
 533
 534 err_out_free:
 535         free(data);
 536 err_out_exit:
 537         return err;
 538 }
 539
 540 int dnet_meta_update_check_status_raw(struct dnet_node *n, struct dnet_meta_container *mc)
 541 {
 542         struct dnet_meta *m = NULL;
 543         struct dnet_meta_check_status *meta_check;
 544         int err = 0;
 545
 546         m = dnet_meta_search(n, mc, DNET_META_CHECK_STATUS);
 547
 548         if (!m) {
 549                 mc->data = realloc(mc->data, mc->size + sizeof(struct dnet_meta) + sizeof(struct dnet_meta_check_status));
 550                 if (!mc->data) {
 551                         err = -ENOMEM;
 552                         goto err_out_free;
 553                 }
 554
 555                 m = mc->data + mc->size;
 556                 mc->size += sizeof(struct dnet_meta) + sizeof(struct dnet_meta_check_status);
 557
 558                 m->type = DNET_META_CHECK_STATUS;
 559                 m->size = sizeof(struct dnet_meta_check_status);
 560                 dnet_convert_meta(m);
 561         }
 562
 563         meta_check = (struct dnet_meta_check_status *)m->data;
 564         meta_check->status = 0;
 565
 566         dnet_current_time(&meta_check->tm);
 567
 568         dnet_convert_meta_check_status(meta_check);
 569
 570         return err;
 571
 572 err_out_free:
 573         free(mc->data);
 574         return err;
 575 }
 576
 577 int dnet_meta_update_check_status(struct dnet_node *n, struct dnet_meta_container *mc)
 578 {
 579         struct dnet_raw_id id;
 580         int err;
 581
 582         err = dnet_meta_update_check_status_raw(n, mc);
 583
 584         if (err) {
 585                 dnet_log(n, DNET_LOG_ERROR, "%s: failed to update DNET_META_CHECK_STATUS, err=%d\n",
 586                                 dnet_dump_id(&mc->id), err);
 587                 return err;
 588         } else {
 589                 memcpy(&id.id, &mc->id.id, DNET_ID_SIZE);
 590                 err = n->cb->meta_write(n->cb->command_private, &id, mc->data, mc->size);
 591                 if (err) {
 592                         dnet_log(n, DNET_LOG_ERROR, "%s: failed to write meta, err=%d\n",
 593                                         dnet_dump_id(&mc->id), err);
 594                 }
 595         }
 596
 597         return err;
 598 }
 599