bindings/cpp/test.cpp

   1 /*
   2  * 2008+ Copyright (c) Evgeniy Polyakov <zbr@ioremap.net>
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  */
  15
  16 #include <sys/time.h>
  17 #include <sys/resource.h>
  18
  19 #include <errno.h>
  20 #include <stdarg.h>
  21 #include <string.h>
  22
  23 #include <sstream>
  24 #include <fstream>
  25
  26 #include <elliptics/cppdef.h>
  27
  28 using namespace ioremap::elliptics;
  29
  30 static void test_log_raw(logger *l, int level, const char *format, ...)
  31 {
  32         va_list args;
  33         char buf[1024];
  34         int buflen = sizeof(buf);
  35
  36         if (l->get_log_level() < level)
  37                 return;
  38
  39         va_start(args, format);
  40         vsnprintf(buf, buflen, format, args);
  41         buf[buflen-1] = '\0';
  42         l->log(level, buf);
  43         va_end(args);
  44 }
  45
  46 class callback_io : public callback {
  47         public:
  48                 callback_io(logger *l) { log = l; };
  49                 virtual ~callback_io() {};
  50
  51                 virtual int             handle(struct dnet_net_state *state, struct dnet_cmd *cmd);
  52
  53         private:
  54                 logger          *log;
  55 };
  56
  57 int callback_io::handle(struct dnet_net_state *state, struct dnet_cmd *cmd)
  58 {
  59         int err;
  60         struct dnet_io_attr *io;
  61
  62         if (is_trans_destroyed(state, cmd)) {
  63                 err = -EINVAL;
  64                 goto err_out_exit;
  65         }
  66
  67         if (cmd->status || !cmd->size) {
  68                 err = cmd->status;
  69                 goto err_out_exit;
  70         }
  71
  72         if (cmd->size <= sizeof(struct dnet_io_attr)) {
  73                 test_log_raw(log, DNET_LOG_ERROR, "%s: read completion error: wrong size: "
  74                                 "cmd_size: %llu, must be more than %zu.\n",
  75                                 dnet_dump_id(&cmd->id), (unsigned long long)cmd->size,
  76                                 sizeof(struct dnet_io_attr));
  77                 err = -EINVAL;
  78                 goto err_out_exit;
  79         }
  80
  81         if (!cmd->size) {
  82                 test_log_raw(log, DNET_LOG_ERROR, "%s: no attributes but command size is not null.\n",
  83                                 dnet_dump_id(&cmd->id));
  84                 err = -EINVAL;
  85                 goto err_out_exit;
  86         }
  87
  88         io = (struct dnet_io_attr *)(cmd + 1);
  89
  90         dnet_convert_io_attr(io);
  91         err = 0;
  92
  93         test_log_raw(log, DNET_LOG_INFO, "%s: io completion: offset: %llu, size: %llu.\n",
  94                         dnet_dump_id(&cmd->id), (unsigned long long)io->offset, (unsigned long long)io->size);
  95
  96 err_out_exit:
  97         if (!cmd || !(cmd->flags & DNET_FLAGS_MORE))
  98                 test_log_raw(log, DNET_LOG_INFO, "%s: io completed: %d.\n", cmd ? dnet_dump_id(&cmd->id) : "nil", err);
  99         return err;
 100 }
 101
 102 static void test_prepare_commit(node &n, int psize, int csize)
 103 {
 104         std::string written, ret;
 105         try {
 106                 std::string key = "prepare-commit-test";
 107
 108                 std::string prepare_data = "prepare data|";
 109                 std::string commit_data = "commit data";
 110                 std::string plain_data[3] = {"plain data0|", "plain data1|", "plain data2|"};
 111
 112                 if (psize)
 113                         prepare_data.clear();
 114                 if (csize)
 115                         commit_data.clear();
 116
 117                 uint64_t offset = 0;
 118                 uint64_t total_size_to_reserve = 1024;
 119
 120                 uint64_t cflags = 0;
 121                 unsigned int ioflags = 0;
 122
 123                 int column = 0;
 124
 125                 n.write_prepare(key, prepare_data, offset, total_size_to_reserve, cflags, ioflags, column);
 126                 offset += prepare_data.size();
 127
 128                 written += prepare_data;
 129
 130                 for (int i = 0; i < 3; ++i) {
 131                         n.write_plain(key, plain_data[i], offset, cflags, ioflags, column);
 132                         offset += plain_data[i].size();
 133
 134                         written += plain_data[i];
 135                 }
 136
 137                 /* append data first so that subsequent written.size() call returned real size of the written data */
 138                 written += commit_data;
 139
 140                 n.write_commit(key, commit_data, offset, written.size(), cflags, ioflags, column);
 141
 142                 ret = n.read_data_wait(key, 0, 0, cflags, ioflags, column);
 143                 std::cout << "prepare/commit write: '" << written << "', read: '" << ret << "'" << std::endl;
 144         } catch (const std::exception &e) {
 145                 std::cerr << "PREPARE/COMMIT test failed: " << e.what() << std::endl;
 146                 throw;
 147         }
 148
 149         if (ret != written) {
 150                 std::cerr << "PREPARE/COMMIT test failed: read mismatch" << std::endl;
 151                 throw std::runtime_error("PREPARE/COMMIT test failed: read mismatch");
 152         }
 153 }
 154
 155 static void test_range_request(node &n, int limit_start, int limit_num, uint64_t cflags, int group_id)
 156 {
 157         struct dnet_io_attr io;
 158
 159         memset(&io, 0, sizeof(io));
 160
 161 #if 0
 162         dnet_parse_numeric_id("76a046fcd25ebeaaa65a0fa692faf8b8701695c6ba67008b5922ae9f134fc1da7ffffed191edf767000000000000000000000000000000000000000000000000", &io.id);
 163         dnet_parse_numeric_id("76a046fcd25ebeaaa65a0fa692faf8b8701695c6ba67008b5922ae9f134fc1da7ffffed22220037fffffffffffffffffffffffffffffffffffffffffffffffff", &io.parent);
 164 #else
 165         memset(io.id, 0x00, sizeof(io.id));
 166         memset(io.parent, 0xff, sizeof(io.id));
 167 #endif
 168         io.start = limit_start;
 169         io.num = limit_num;
 170
 171         std::vector<std::string> ret;
 172         ret = n.read_data_range(io, group_id, cflags);
 173
 174         std::cout << "range [LIMIT(" << limit_start << ", " << limit_num << "): " << ret.size() << " elements" << std::endl;
 175 #if 0
 176         for (size_t i = 0; i < ret.size(); ++i) {
 177                 char id_str[DNET_ID_SIZE * 2 + 1];
 178                 const char *data = ret[i].data();
 179                 const unsigned char *id = (const unsigned char *)data;
 180                 uint64_t size = dnet_bswap64(*(uint64_t *)(data + DNET_ID_SIZE));
 181                 char *str = (char *)(data + DNET_ID_SIZE + 8);
 182
 183                 std::cout << "range [LIMIT(" << limit_start << ", " << limit_num << "): " <<
 184                         dnet_dump_id_len_raw(id, DNET_ID_SIZE, id_str) << ": size: " << size << ": " << str << std::endl;
 185         }
 186 #endif
 187 }
 188
 189 static void test_lookup_parse(const std::string &key, const std::string &lret)
 190 {
 191         struct dnet_addr *addr = (struct dnet_addr *)lret.data();
 192         struct dnet_cmd *cmd = (struct dnet_cmd *)(addr + 1);
 193         struct dnet_addr_attr *a = (struct dnet_addr_attr *)(cmd + 1);
 194
 195         dnet_convert_addr_attr(a);
 196         std::cout << key << ": lives on addr: " << dnet_server_convert_dnet_addr(&a->addr);
 197
 198         if (cmd->size > sizeof(struct dnet_addr_attr)) {
 199                 struct dnet_file_info *info = (struct dnet_file_info *)(a + 1);
 200
 201                 dnet_convert_file_info(info);
 202                 std::cout << ": mode: " << std::oct << info->mode << std::dec;
 203                 std::cout << ", offset: " << (unsigned long long)info->offset;
 204                 std::cout << ", size: " << (unsigned long long)info->size;
 205                 std::cout << ", file: " << (char *)(info + 1);
 206         }
 207         std::cout << std::endl;
 208 }
 209
 210 static void test_lookup(node &n, std::vector<int> &groups)
 211 {
 212         try {
 213                 std::string key = "2.xml";
 214                 std::string data = "lookup data";
 215
 216                 std::string lret = n.write_data_wait(key, data, 0, 0, 0, 0);
 217                 test_lookup_parse(key, lret);
 218
 219                 struct dnet_id id;
 220                 n.transform(key, id);
 221                 id.group_id = 0;
 222                 id.type = 0;
 223
 224                 uint64_t cflags = 0;
 225
 226                 struct timespec ts = {0, 0};
 227                 n.write_metadata(id, key, groups, ts, cflags);
 228
 229                 lret = n.lookup(key);
 230                 test_lookup_parse(key, lret);
 231         } catch (const std::exception &e) {
 232                 std::cerr << "LOOKUP test failed: " << e.what() << std::endl;
 233         }
 234 }
 235
 236 static void test_append(node &n)
 237 {
 238         try {
 239                 std::string key = "append-test";
 240                 std::string data = "first part of the message";
 241
 242                 n.write_data_wait(key, data, 0, 0, 0, 0);
 243
 244                 data = "| second part of the message";
 245                 n.write_data_wait(key, data, 0, 0, DNET_IO_FLAGS_APPEND, 0);
 246
 247                 std::cout << key << ": " << n.read_data_wait(key, 0, 0, 0, 0, 0) << std::endl;
 248         } catch (const std::exception &e) {
 249                 std::cerr << "APPEND test failed: " << e.what() << std::endl;
 250                 throw std::runtime_error("APPEND test failed");
 251         }
 252 }
 253
 254 static void read_column_raw(node &n, const std::string &key, const std::string &data, int column)
 255 {
 256         std::string ret;
 257         try {
 258                 ret = n.read_data_wait(key, 0, 0, 0, 0, column);
 259         } catch (const std::exception &e) {
 260                 std::cerr << "COLUMN-" << column << " read test failed: " << e.what() << std::endl;
 261                 throw;
 262         }
 263
 264         std::cout << "read-column-" << column << ": " << key << " : " << ret << std::endl;
 265         if (ret != data) {
 266                 throw std::runtime_error("column test failed");
 267         }
 268 }
 269
 270 static void column_test(node &n)
 271 {
 272         std::string key = "some-key-1";
 273
 274         std::string data0 = "some-compressed-data-in-column-0";
 275         std::string data1 = "some-data-in-column-2";
 276         std::string data2 = "some-data-in-column-3";
 277
 278         n.write_data_wait(key, data0, 0, 0, DNET_IO_FLAGS_COMPRESS, 0);
 279         n.write_data_wait(key, data1, 0, 0, 0, 2);
 280         n.write_data_wait(key, data2, 0, 0, 0, 3);
 281
 282         read_column_raw(n, key, data0, 0);
 283         read_column_raw(n, key, data1, 2);
 284         read_column_raw(n, key, data2, 3);
 285 }
 286
 287 static void test_bulk_write(node &n)
 288 {
 289         try {
 290                 std::vector<struct dnet_io_attr> ios;
 291                 std::vector<std::string> data;
 292
 293                 int i;
 294
 295                 for (i = 0; i < 3; ++i) {
 296                         std::ostringstream os;
 297                         struct dnet_io_attr io;
 298                         struct dnet_id id;
 299
 300                         os << "bulk_write" << i;
 301
 302                         memset(&io, 0, sizeof(io));
 303                         memset(&id, 0, sizeof(id));
 304
 305                         n.transform(os.str(), id);
 306                         memcpy(io.id, id.id, DNET_ID_SIZE);
 307                         io.type = id.type;
 308                         io.size = os.str().size();
 309
 310                         ios.push_back(io);
 311                         data.push_back(os.str());
 312                 }
 313
 314                 std::string ret = n.bulk_write(ios, data, 0);
 315
 316                 std::cout << "ret size = " << ret.size() << std::endl;
 317
 318                 uint64_t cflags = 0;
 319                 int ioflags = DNET_IO_FLAGS_NOCSUM;
 320                 int type = 0;
 321
 322                 uint64_t offset = 0;
 323                 uint64_t size = 0;
 324
 325                 /* read without checksums since we did not write metadata */
 326                 for (i = 0; i < 3; ++i) {
 327                         std::ostringstream os;
 328
 329                         os << "bulk_write" << i;
 330                         std::cout << os.str() << ": " << n.read_data_wait(os.str(), offset, size, cflags, ioflags, type) << std::endl;
 331                 }
 332         } catch (const std::exception &e) {
 333                 std::cerr << "BULK WRITE test failed: " << e.what() << std::endl;
 334         }
 335 }
 336
 337 static void test_bulk_read(node &n)
 338 {
 339         try {
 340                 std::vector<std::string> keys;
 341
 342                 int i;
 343
 344                 for (i = 0; i < 3; ++i) {
 345                         std::ostringstream os;
 346                         os << "bulk_write" << i;
 347                         keys.push_back(os.str());
 348                 }
 349
 350                 std::vector<std::string> ret = n.bulk_read(keys, 0);
 351
 352                 std::cout << "ret size = " << ret.size() << std::endl;
 353
 354                 /* read without checksums since we did not write metadata */
 355                 for (i = 0; i < 3; ++i) {
 356                         std::ostringstream os;
 357
 358                         os << "bulk_read" << i;
 359                         std::cout << os.str() << ": " << ret[i].substr(DNET_ID_SIZE + 8) << std::endl;
 360                 }
 361         } catch (const std::exception &e) {
 362                 std::cerr << "BULK READ test failed: " << e.what() << std::endl;
 363         }
 364
 365 }
 366
 367 static void memory_test_io(node &n, int num)
 368 {
 369         int ids[16];
 370
 371         for (int i = 0; i < num; ++i) {
 372                 std::string data;
 373
 374                 data.resize(rand() % 102400 + 100);
 375
 376                 for (int j = 0; j < (int)ARRAY_SIZE(ids); ++j)
 377                         ids[j] = rand();
 378
 379                 std::string id((char *)ids, sizeof(ids));
 380                 std::string written;
 381
 382                 try {
 383                         written = n.write_data_wait(id, data, 0, 0, 0, 0);
 384                         std::string res = n.read_data_wait(id, 0, 0, 0, 0, 0);
 385                 } catch (const std::exception &e) {
 386                         std::cerr << "could not perform read/write: " << e.what() << std::endl;
 387                         if (written.size() > 0) {
 388                                 std::cerr << "but written successfully\n";
 389                                 test_lookup_parse(id, written);
 390                         }
 391                 }
 392         }
 393
 394 }
 395
 396 static void test_cache_write(node &n, int num)
 397 {
 398         try {
 399                 std::vector<struct dnet_io_attr> ios;
 400                 std::vector<std::string> data;
 401
 402                 int i;
 403
 404                 for (i = 0; i < num; ++i) {
 405                         std::ostringstream os;
 406                         struct dnet_io_attr io;
 407                         struct dnet_id id;
 408
 409                         os << "test_cache" << i;
 410
 411                         memset(&io, 0, sizeof(io));
 412                         memset(&id, 0, sizeof(id));
 413
 414                         n.transform(os.str(), id);
 415                         memcpy(io.id, id.id, DNET_ID_SIZE);
 416                         io.type = id.type;
 417                         io.size = os.str().size();
 418
 419                         ios.push_back(io);
 420                         data.push_back(os.str());
 421                 }
 422
 423                 n.bulk_write(ios, data, 0);
 424         } catch (const std::exception &e) {
 425                 std::cerr << "cache write test failed: " << e.what() << std::endl;
 426         }
 427         std::cout << "Cache entries writted: " << num << std::endl;
 428 }
 429
 430 static void test_cache_read(node &n, int num)
 431 {
 432         int count = 0;
 433
 434         /* Read random 20% of records written by test_cache_write() */
 435         for (int i = 0; i < num; ++i) {
 436                 if ((rand() % 100) > 20)
 437                         continue;
 438
 439                 std::ostringstream os;
 440
 441                 os << "test_cache" << i;
 442
 443                 std::string id(os.str());
 444
 445                 uint64_t cflags = 0;
 446                 int ioflags = DNET_IO_FLAGS_NOCSUM;
 447                 int type = 0;
 448
 449                 uint64_t offset = 0;
 450                 uint64_t size = 0;
 451
 452                 try {
 453                         n.read_data_wait(os.str(), offset, size, cflags, ioflags, type);
 454                 } catch (const std::exception &e) {
 455                         std::cerr << "could not perform read : " << e.what() << std::endl;
 456                 }
 457                 count++;
 458         }
 459         std::cout << "Cache entries read: " << count << std::endl;
 460 }
 461
 462 static void test_cache_delete(node &n, int num)
 463 {
 464         int count = 0;
 465
 466         /* Read random 20% of records written by test_cache_write() */
 467         for (int i = 0; i < num; ++i) {
 468                 if ((rand() % 100) > 20)
 469                         continue;
 470
 471                 std::ostringstream os;
 472
 473                 os << "test_cache" << i;
 474
 475                 std::string id(os.str());
 476
 477                 try {
 478                         n.remove(id, 0);
 479                 } catch (const std::exception &e) {
 480                         std::cerr << "could not perform remove: " << e.what() << std::endl;
 481                 }
 482                 count++;
 483         }
 484         std::cout << "Cache entries deleted: " << count << std::endl;
 485 }
 486
 487 static void memory_test(node &n)
 488 {
 489         struct rusage start, end;
 490
 491         getrusage(RUSAGE_SELF, &start);
 492         memory_test_io(n, 1000);
 493         getrusage(RUSAGE_SELF, &end);
 494         std::cout << "IO leaked: " << end.ru_maxrss - start.ru_maxrss << " Kb\n";
 495 }
 496
 497 void usage(char *p)
 498 {
 499         fprintf(stderr, "Usage: %s <options>\n"
 500                         "  -r host              - remote host name\n"
 501                         "  -p port              - remote port\n"
 502                         "  -g group_id          - group_id for range request and bulk write\n"
 503                         "  -w                   - write cache before read\n"
 504                         "  -m                   - start client's memory leak test (rather long - several minutes, and space consuming)\n"
 505                         , p);
 506         exit(-1);
 507 }
 508
 509 int main(int argc, char *argv[])
 510 {
 511         int g[] = {1, 2, 3};
 512         std::vector<int> groups(g, g+ARRAY_SIZE(g));
 513         char *host = (char *)"localhost";
 514         int port = 1025;
 515         int ch, write_cache = 0;
 516         int mem_check = 0;
 517         int group_id = 2;
 518
 519         while ((ch = getopt(argc, argv, "mr:p:g:wh")) != -1) {
 520                 switch (ch) {
 521                         case 'r':
 522                                 host = optarg;
 523                                 break;
 524                         case 'p':
 525                                 port = atoi(optarg);
 526                                 break;
 527                         case 'g':
 528                                 group_id = atoi(optarg);
 529                                 break;
 530                         case 'w':
 531                                 write_cache = 1;
 532                                 break;
 533                         case 'm':
 534                                 mem_check = 1;
 535                                 break;
 536                         case 'h':
 537                         default:
 538                                 usage(argv[0]);
 539                 }
 540         }
 541
 542
 543         try {
 544                 log_file log("/dev/stderr", DNET_LOG_ERROR);
 545
 546                 node n(log);
 547                 n.add_groups(groups);
 548
 549                 try {
 550                         n.add_remote(host, port, AF_INET);
 551                 } catch (...) {
 552                         throw std::runtime_error("Could not add remote nodes, exiting");
 553                 }
 554
 555                 test_lookup(n, groups);
 556
 557                 n.stat_log();
 558
 559                 column_test(n);
 560
 561                 test_prepare_commit(n, 0, 0);
 562                 test_prepare_commit(n, 1, 0);
 563                 test_prepare_commit(n, 0, 1);
 564                 test_prepare_commit(n, 1, 1);
 565
 566                 test_range_request(n, 0, 0, 0, group_id);
 567                 test_range_request(n, 0, 0, DNET_ATTR_SORT, group_id);
 568                 test_range_request(n, 1, 0, 0, group_id);
 569                 test_range_request(n, 0, 1, 0, group_id);
 570
 571                 test_append(n);
 572
 573                 test_bulk_write(n);
 574                 test_bulk_read(n);
 575
 576                 if (mem_check)
 577                         memory_test(n);
 578
 579                 if (write_cache)
 580                         test_cache_write(n, 1000);
 581
 582                 test_cache_read(n, 1000);
 583                 test_cache_delete(n, 1000);
 584                 test_cache_write(n, 1000);
 585
 586         } catch (const std::exception &e) {
 587                 std::cerr << "Error occured : " << e.what() << std::endl;
 588         } catch (int err) {
 589                 std::cerr << "Error : " << err << std::endl;
 590         }
 591 }