Fixed reply size calculation
[elliptics.git] / bindings / python / elliptics_python.cpp
blob8faacb73c0c266e2d151298c3e078daea030aa10
1 /*
2 * 2008+ Copyright (c) Evgeniy Polyakov <zbr@ioremap.net>
3 * All rights reserved.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
16 #include <netdb.h>
17 #include <boost/python.hpp>
18 #include <boost/python/list.hpp>
19 #include <boost/python/dict.hpp>
21 #include <elliptics/cppdef.h>
23 using namespace boost::python;
24 using namespace ioremap::elliptics;
26 enum elliptics_cflags {
27 cflags_default = 0,
28 cflags_direct = DNET_FLAGS_DIRECT,
29 cflags_nolock = DNET_FLAGS_NOLOCK,
32 enum elliptics_ioflags {
33 ioflags_default = 0,
34 ioflags_append = DNET_IO_FLAGS_APPEND,
35 ioflags_compress = DNET_IO_FLAGS_COMPRESS,
36 ioflags_meta = DNET_IO_FLAGS_META,
37 ioflags_prepare = DNET_IO_FLAGS_PREPARE,
38 ioflags_commit = DNET_IO_FLAGS_COMMIT,
39 ioflags_overwrite = DNET_IO_FLAGS_OVERWRITE,
40 ioflags_nocsum = DNET_IO_FLAGS_NOCSUM,
41 ioflags_plain_write = DNET_IO_FLAGS_PLAIN_WRITE,
42 ioflags_cache = DNET_IO_FLAGS_CACHE,
43 ioflags_cache_only = DNET_IO_FLAGS_CACHE_ONLY,
44 ioflags_cache_remove_from_disk = DNET_IO_FLAGS_CACHE_REMOVE_FROM_DISK,
47 enum elliptics_log_level {
48 log_level_data = DNET_LOG_DATA,
49 log_level_error = DNET_LOG_ERROR,
50 log_level_info = DNET_LOG_INFO,
51 log_level_notice = DNET_LOG_NOTICE,
52 log_level_debug = DNET_LOG_DEBUG,
55 static void elliptics_extract_arr(const list &l, unsigned char *dst, int *dlen)
57 int length = len(l);
59 if (length > *dlen)
60 length = *dlen;
62 memset(dst, 0, *dlen);
63 for (int i = 0; i < length; ++i)
64 dst[i] = extract<unsigned char>(l[i]);
67 struct elliptics_id {
68 elliptics_id() : group_id(0), type(0) {}
69 elliptics_id(list id_, int group_, int type_) : id(id_), group_id(group_), type(type_) {}
71 elliptics_id(struct dnet_id &dnet) {
72 for (unsigned int i = 0; i < sizeof(dnet.id); ++i)
73 id.append(dnet.id[i]);
75 group_id = dnet.group_id;
76 type = dnet.type;
79 struct dnet_id to_dnet() const {
80 struct dnet_id dnet;
81 int len = sizeof(dnet.id);
83 elliptics_extract_arr(id, dnet.id, &len);
85 dnet.group_id = group_id;
86 dnet.type = type;
88 return dnet;
91 list id;
92 uint32_t group_id;
93 int type;
96 struct elliptics_range {
97 elliptics_range() : offset(0), size(0),
98 limit_start(0), limit_num(0), cflags(0), ioflags(0), group_id(0), type(0) {}
100 list start, end;
101 uint64_t offset, size;
102 uint64_t limit_start, limit_num;
103 uint64_t cflags;
104 uint32_t ioflags;
105 int group_id;
106 int type;
109 static void elliptics_extract_range(const struct elliptics_range &r, struct dnet_io_attr &io)
111 int len = sizeof(io.id);
113 elliptics_extract_arr(r.start, io.id, &len);
114 elliptics_extract_arr(r.end, io.parent, &len);
116 io.flags = r.ioflags;
117 io.size = r.size;
118 io.offset = r.offset;
119 io.start = r.limit_start;
120 io.num = r.limit_num;
121 io.type = r.type;
124 class elliptics_log_wrap : public logger, public wrapper<logger> {
125 public:
126 elliptics_log_wrap(const int level = DNET_LOG_INFO) : logger(level) {};
128 void log(const int level, const char *msg) {
129 this->get_override("log")(level, msg);
132 unsigned long clone(void) {
133 return this->get_override("clone")();
137 class elliptics_log_file_wrap : public log_file, public wrapper<log_file> {
138 public:
139 elliptics_log_file_wrap(const char *file, const int level = DNET_LOG_INFO) :
140 log_file(file, level) {};
142 void log(const int level, const char *msg) {
143 if (override log = this->get_override("log")) {
144 log_file::log(level, msg);
145 return;
148 log_file::log(level, msg);
151 void default_log(const int level, const char *msg) { this->log(level, msg); }
153 unsigned long clone(void) {
154 if (override clone = this->get_override("clone"))
155 return log_file::clone();
157 return log_file::clone();
160 unsigned long default_clone(void) { return this->clone(); }
163 class elliptics_config {
164 public:
165 elliptics_config() {
166 memset(&config, 0, sizeof(struct dnet_config));
169 std::string cookie_get(void) const {
170 std::string ret;
171 ret.assign(config.cookie, sizeof(config.cookie));
172 return ret;
175 void cookie_set(const std::string &cookie) {
176 size_t sz = sizeof(config.cookie);
177 if (cookie.size() + 1 < sz)
178 sz = cookie.size() + 1;
179 memset(config.cookie, 0, sizeof(config.cookie));
180 snprintf(config.cookie, sz, "%s", (char *)cookie.data());
183 struct dnet_config config;
186 class elliptics_node_python : public node {
187 public:
188 elliptics_node_python(logger &l) : node(l) {}
190 elliptics_node_python(logger &l, elliptics_config &cfg) : node(l, cfg.config) {}
192 void add_groups(const list &pgroups) {
193 std::vector<int> groups;
195 for (int i=0; i<len(pgroups); ++i)
196 groups.push_back(extract<int>(pgroups[i]));
198 node::add_groups(groups);
201 boost::python::list get_groups(){
202 std::vector<int> groups = node::get_groups();
203 boost::python::list res;
204 for(size_t i=0; i<groups.size(); i++)
206 res.append(groups[i]);
209 return res;
212 void write_metadata_by_id(const struct elliptics_id &id, const std::string &remote, const list &pgroups, uint64_t cflags) {
213 struct timespec ts;
214 memset(&ts, 0, sizeof(ts));
216 struct dnet_id raw = id.to_dnet();
218 std::vector<int> groups;
220 for (int i=0; i<len(pgroups); ++i)
221 groups.push_back(extract<int>(pgroups[i]));
223 write_metadata((const dnet_id&)raw, remote, groups, ts, cflags);
226 void write_metadata_by_data_transform(const std::string &remote, uint64_t cflags) {
227 struct timespec ts;
228 memset(&ts, 0, sizeof(ts));
230 struct dnet_id raw;
232 transform(remote, raw);
234 write_metadata((const dnet_id&)raw, remote, groups, ts, cflags);
237 void read_file_by_id(struct elliptics_id &id, const std::string &file, uint64_t offset, uint64_t size) {
238 struct dnet_id raw = id.to_dnet();
239 read_file(raw, file, offset, size);
242 void read_file_by_data_transform(const std::string &remote, const std::string &file,
243 uint64_t offset, uint64_t size, int type) {
244 read_file(remote, file, offset, size, type);
247 void write_file_by_id(struct elliptics_id &id, const std::string &file,
248 uint64_t local_offset, uint64_t offset, uint64_t size,
249 uint64_t cflags, unsigned int ioflags) {
250 struct dnet_id raw = id.to_dnet();
251 write_file(raw, file, local_offset, offset, size, cflags, ioflags);
254 void write_file_by_data_transform(const std::string &remote, const std::string &file,
255 uint64_t local_offset, uint64_t offset, uint64_t size,
256 uint64_t cflags, unsigned int ioflags, int type) {
257 write_file(remote, file, local_offset, offset, size, cflags, ioflags, type);
260 std::string read_data_by_id(const struct elliptics_id &id, uint64_t offset, uint64_t size,
261 uint64_t cflags, unsigned int ioflags) {
262 struct dnet_id raw = id.to_dnet();
263 return read_data_wait(raw, offset, size, cflags, ioflags);
266 std::string read_data_by_data_transform(const std::string &remote, uint64_t offset, uint64_t size,
267 uint64_t cflags, unsigned int ioflags, int type) {
268 return read_data_wait(remote, offset, size, cflags, ioflags, type);
271 list prepare_latest_by_id(const struct elliptics_id &id, uint64_t cflags, list gl) {
272 struct dnet_id raw = id.to_dnet();
274 std::vector<int> groups;
275 for (int i = 0; i < len(gl); ++i)
276 groups.push_back(extract<int>(gl[i]));
278 prepare_latest(raw, cflags, groups);
280 list l;
281 for (unsigned i = 0; i < groups.size(); ++i)
282 l.append(groups[i]);
284 return l;
287 std::string prepare_latest_by_id_str(const struct elliptics_id &id, uint64_t cflags, list gl) {
288 struct dnet_id raw = id.to_dnet();
290 std::vector<int> groups;
291 for (int i = 0; i < len(gl); ++i)
292 groups.push_back(extract<int>(gl[i]));
294 prepare_latest(raw, cflags, groups);
296 std::string ret;
297 ret.assign((char *)groups.data(), groups.size() * 4);
299 return ret;
302 std::string read_latest_by_id(const struct elliptics_id &id, uint64_t offset, uint64_t size,
303 uint64_t cflags, unsigned int ioflags) {
304 struct dnet_id raw = id.to_dnet();
305 return read_latest(raw, offset, size, cflags, ioflags);
308 std::string read_latest_by_data_transform(const std::string &remote, uint64_t offset, uint64_t size,
309 uint64_t cflags, unsigned int ioflags, int type) {
310 return read_latest(remote, offset, size, cflags, ioflags, type);
313 std::string write_data_by_id(const struct elliptics_id &id, const std::string &data, uint64_t remote_offset,
314 uint64_t cflags, unsigned int ioflags) {
315 struct dnet_id raw = id.to_dnet();
316 return write_data_wait(raw, data, remote_offset, cflags, ioflags);
319 std::string write_data_by_data_transform(const std::string &remote, const std::string &data, uint64_t remote_offset,
320 uint64_t cflags, unsigned int ioflags, int type) {
321 return write_data_wait(remote, data, remote_offset, cflags, ioflags, type);
324 std::string write_cache_by_id(const struct elliptics_id &id, const std::string &data,
325 uint64_t cflags, unsigned int ioflags, long timeout) {
326 struct dnet_id raw = id.to_dnet();
327 raw.type = 0;
328 return write_cache(raw, data, cflags, ioflags, timeout);
331 std::string write_cache_by_data_transform(const std::string &remote, const std::string &data,
332 uint64_t cflags, unsigned int ioflags, long timeout) {
333 return write_cache(remote, data, cflags, ioflags, timeout);
336 std::string lookup_addr_by_data_transform(const std::string &remote, const int group_id) {
337 return lookup_addr(remote, group_id);
340 std::string lookup_addr_by_id(const struct elliptics_id &id) {
341 struct dnet_id raw = id.to_dnet();
343 return lookup_addr(raw);
346 boost::python::tuple parse_lookup(const std::string &lookup) {
347 const void *data = lookup.data();
349 struct dnet_addr *addr = (struct dnet_addr *)data;
350 struct dnet_cmd *cmd = (struct dnet_cmd *)(addr + 1);
351 struct dnet_addr_attr *a = (struct dnet_addr_attr *)(cmd + 1);
352 struct dnet_file_info *info = (struct dnet_file_info *)(a + 1);
353 dnet_convert_file_info(info);
355 std::string address(dnet_server_convert_dnet_addr(addr));
356 int port = dnet_server_convert_port((struct sockaddr *)a->addr.addr, a->addr.addr_len);
358 return make_tuple(address, port, info->size);
361 boost::python::tuple lookup_by_data_transform(const std::string &remote) {
362 return parse_lookup(lookup(remote));
365 boost::python::tuple lookup_by_id(const struct elliptics_id &id) {
366 struct dnet_id raw = id.to_dnet();
368 return parse_lookup(lookup(raw));
371 struct dnet_node_status update_status_by_id(const struct elliptics_id &id, struct dnet_node_status &status) {
372 struct dnet_id raw = id.to_dnet();
374 update_status(raw, &status);
375 return status;
378 struct dnet_node_status update_status_by_string(const std::string &saddr, const int port, const int family,
379 struct dnet_node_status &status) {
380 update_status(saddr.c_str(), port, family, &status);
381 return status;
384 boost::python::list read_data_range(const struct elliptics_range &r) {
385 struct dnet_io_attr io;
386 elliptics_extract_range(r, io);
388 std::vector<std::string> ret;
389 ret = node::read_data_range(io, r.group_id, r.cflags);
391 boost::python::list l;
393 for (size_t i = 0; i < ret.size(); ++i) {
394 l.append(ret[i]);
397 return l;
400 boost::python::list get_routes() {
402 std::vector<std::pair<struct dnet_id, struct dnet_addr> > routes;
403 std::vector<std::pair<struct dnet_id, struct dnet_addr> >::iterator it;
405 boost::python::list res;
407 routes = node::get_routes();
409 for (it = routes.begin(); it != routes.end(); it++) {
410 struct elliptics_id id(it->first);
411 std::string address(dnet_server_convert_dnet_addr(&(it->second)));
413 res.append(make_tuple(id, address));
416 return res;
419 std::string exec_name(const struct elliptics_id &id, const std::string &event,
420 const std::string &data, const std::string &binary) {
421 struct dnet_id raw = id.to_dnet();
423 return exec_locked(&raw, event, data, binary);
426 std::string exec_name_by_name(const std::string &remote, const std::string &event,
427 const std::string &data, const std::string &binary) {
428 struct dnet_id raw;
429 transform(remote, raw);
430 raw.type = 0;
431 raw.group_id = 0;
433 return exec_locked(&raw, event, data, binary);
436 std::string exec_name_all(const std::string &event, const std::string &data, const std::string &binary) {
437 return exec_locked(NULL, event, data, binary);
440 void remove_by_id(const struct elliptics_id &id, uint64_t cflags, uint64_t ioflags) {
441 struct dnet_id raw = id.to_dnet();
443 remove_raw(raw, cflags, ioflags);
446 void remove_by_name(const std::string &remote, uint64_t cflags, uint64_t ioflags, int type) {
447 remove_raw(remote, cflags, ioflags, type);
450 list bulk_read_by_name(const list &keys, uint64_t cflags = 0) {
451 unsigned int length = len(keys);
453 std::vector<std::string> k;
454 k.resize(length);
456 for (unsigned int i = 0; i < length; ++i)
457 k[i] = extract<std::string>(keys[i]);
459 std::vector<std::string> ret = bulk_read(k, cflags);
461 list py_ret;
462 for (size_t i = 0; i < ret.size(); ++i) {
463 py_ret.append(ret[i]);
466 return py_ret;
469 list stat_log() {
470 list statistics;
471 callback c;
472 std::string ret;
473 int err;
474 int i;
476 err = dnet_request_stat(m_node, NULL, DNET_CMD_STAT_COUNT, DNET_ATTR_CNTR_GLOBAL,
477 callback::complete_callback, (void *)&c);
478 if (err < 0) {
479 std::ostringstream str;
480 str << "Failed to request statistics: " << err;
481 throw std::runtime_error(str.str());
484 ret = c.wait(err);
486 const void *data = ret.data();
487 int size = ret.size();
489 while (size > 0) {
490 dict node_stat, storage_commands, proxy_commands, counters;
491 struct dnet_addr *addr = (struct dnet_addr *)data;
492 struct dnet_cmd *cmd = (struct dnet_cmd *)(addr + 1);
493 if (cmd->size <= sizeof(struct dnet_addr_stat)) {
494 size -= cmd->size + sizeof(struct dnet_addr) + sizeof(struct dnet_cmd);
495 data = (char *)data + cmd->size + sizeof(struct dnet_addr) + sizeof(struct dnet_cmd);
496 continue;
499 struct dnet_addr_stat *as = (struct dnet_addr_stat *)(cmd + 1);
501 dnet_convert_addr_stat(as, 0);
502 std::string address(dnet_server_convert_dnet_addr(addr));
503 node_stat[std::string("addr")] = address;
504 node_stat[std::string("group_id")] = cmd->id.group_id;
506 for (i = 0; i < as->num; ++i) {
507 if (i < as->cmd_num) {
508 storage_commands[std::string(dnet_counter_string(i, as->cmd_num))] =
509 make_tuple((unsigned long long)as->count[i].count, (unsigned long long)as->count[i].err);
510 } else if (i < (as->cmd_num * 2)) {
511 proxy_commands[std::string(dnet_counter_string(i, as->cmd_num))] =
512 make_tuple((unsigned long long)as->count[i].count, (unsigned long long)as->count[i].err);
513 } else {
514 counters[std::string(dnet_counter_string(i, as->cmd_num))] =
515 make_tuple((unsigned long long)as->count[i].count, (unsigned long long)as->count[i].err);
519 node_stat["storage_commands"] = storage_commands;
520 node_stat["proxy_commands"] = proxy_commands;
521 node_stat["counters"] = counters;
523 statistics.append(node_stat);
525 int sz = sizeof(struct dnet_addr) + sizeof(struct dnet_cmd) + cmd->size;
526 size -= sz;
527 data = (char *)data + sz;
530 return statistics;
535 BOOST_PYTHON_MEMBER_FUNCTION_OVERLOADS(add_remote_overloads, add_remote, 2, 3);
537 BOOST_PYTHON_MODULE(libelliptics_python) {
538 class_<elliptics_id>("elliptics_id", init<>())
539 .def(init<list, int, int>())
540 .def_readwrite("id", &elliptics_id::id)
541 .def_readwrite("group_id", &elliptics_id::group_id)
542 .def_readwrite("type", &elliptics_id::type)
545 class_<elliptics_range>("elliptics_range", init<>())
546 .def_readwrite("start", &elliptics_range::start)
547 .def_readwrite("end", &elliptics_range::end)
548 .def_readwrite("offset", &elliptics_range::offset)
549 .def_readwrite("size", &elliptics_range::size)
550 .def_readwrite("ioflags", &elliptics_range::ioflags)
551 .def_readwrite("cflags", &elliptics_range::cflags)
552 .def_readwrite("group_id", &elliptics_range::group_id)
553 .def_readwrite("type", &elliptics_range::type)
554 .def_readwrite("limit_start", &elliptics_range::limit_start)
555 .def_readwrite("limit_num", &elliptics_range::limit_num)
558 class_<elliptics_log_wrap, boost::noncopyable>("elliptics_log", init<const uint32_t>())
559 .def("log", pure_virtual(&logger::log))
560 .def("clone", pure_virtual(&logger::clone))
563 class_<elliptics_log_file_wrap, boost::noncopyable, bases<logger> >("elliptics_log_file", init<const char *, const uint32_t>())
564 .def("log", &log_file::log, &elliptics_log_file_wrap::default_log)
565 .def("clone", &log_file::clone, &elliptics_log_file_wrap::default_clone)
568 class_<dnet_node_status>("dnet_node_status", init<>())
569 .def_readwrite("nflags", &dnet_node_status::nflags)
570 .def_readwrite("status_flags", &dnet_node_status::status_flags)
571 .def_readwrite("log_level", &dnet_node_status::log_level)
574 class_<node>("elliptics_node", init<logger &>())
575 .def("add_remote", &node::add_remote, add_remote_overloads())
578 class_<dnet_config>("dnet_config", init<>())
579 .def_readwrite("wait_timeout", &dnet_config::wait_timeout)
580 .def_readwrite("flags", &dnet_config::flags)
581 .def_readwrite("check_timeout", &dnet_config::check_timeout)
582 .def_readwrite("io_thread_num", &dnet_config::io_thread_num)
583 .def_readwrite("nonblocking_io_thread_num", &dnet_config::nonblocking_io_thread_num)
584 .def_readwrite("net_thread_num", &dnet_config::net_thread_num)
585 .def_readwrite("client_prio", &dnet_config::client_prio)
588 class_<elliptics_config>("elliptics_config", init<>())
589 .def_readwrite("config", &elliptics_config::config)
590 .add_property("cookie", &elliptics_config::cookie_get, &elliptics_config::cookie_set)
593 class_<elliptics_node_python, bases<node> >("elliptics_node_python", init<logger &>())
594 .def(init<logger &, elliptics_config &>())
595 .def("add_remote", &node::add_remote, add_remote_overloads())
597 .def("add_groups", &elliptics_node_python::add_groups)
598 .def("get_groups", &elliptics_node_python::get_groups)
600 .def("read_file", &elliptics_node_python::read_file_by_id)
601 .def("read_file", &elliptics_node_python::read_file_by_data_transform)
602 .def("write_file", &elliptics_node_python::write_file_by_id)
603 .def("write_file", &elliptics_node_python::write_file_by_data_transform)
605 .def("read_data", &elliptics_node_python::read_data_by_id)
606 .def("read_data", &elliptics_node_python::read_data_by_data_transform)
608 .def("prepare_latest", &elliptics_node_python::prepare_latest_by_id)
609 .def("prepare_latest_str", &elliptics_node_python::prepare_latest_by_id_str)
611 .def("read_latest", &elliptics_node_python::read_latest_by_id)
612 .def("read_latest", &elliptics_node_python::read_latest_by_data_transform)
614 .def("write_data", &elliptics_node_python::write_data_by_id)
615 .def("write_data", &elliptics_node_python::write_data_by_data_transform)
617 .def("write_metadata", &elliptics_node_python::write_metadata_by_id)
618 .def("write_metadata", &elliptics_node_python::write_metadata_by_data_transform)
620 .def("write_cache", &elliptics_node_python::write_cache_by_id)
621 .def("write_cache", &elliptics_node_python::write_cache_by_data_transform)
623 .def("lookup_addr", &elliptics_node_python::lookup_addr_by_data_transform)
624 .def("lookup_addr", &elliptics_node_python::lookup_addr_by_id)
626 .def("lookup", &elliptics_node_python::lookup_by_data_transform)
627 .def("lookup", &elliptics_node_python::lookup_by_id)
629 .def("update_status", &elliptics_node_python::update_status_by_id)
630 .def("update_status", &elliptics_node_python::update_status_by_string)
632 .def("read_data_range", &elliptics_node_python::read_data_range)
634 .def("get_routes", &elliptics_node_python::get_routes)
635 .def("stat_log", &elliptics_node_python::stat_log)
637 .def("exec_event", &elliptics_node_python::exec_name)
638 .def("exec_event", &elliptics_node_python::exec_name_by_name)
639 .def("exec_event", &elliptics_node_python::exec_name_all)
641 .def("remove", &elliptics_node_python::remove_by_id)
642 .def("remove", &elliptics_node_python::remove_by_name)
644 .def("bulk_read", &elliptics_node_python::bulk_read_by_name)
647 enum_<elliptics_cflags>("command_flags")
648 .value("default", cflags_default)
649 .value("direct", cflags_direct)
650 .value("nolock", cflags_nolock)
653 enum_<elliptics_ioflags>("io_flags")
654 .value("default", ioflags_default)
655 .value("append", ioflags_append)
656 .value("compress", ioflags_compress)
657 .value("meta", ioflags_meta)
658 .value("prepare", ioflags_prepare)
659 .value("commit", ioflags_commit)
660 .value("overwrite", ioflags_overwrite)
661 .value("nocsum", ioflags_nocsum)
662 .value("plain_write", ioflags_plain_write)
663 .value("nodata", ioflags_plain_write)
664 .value("cache", ioflags_cache)
665 .value("cache_only", ioflags_cache_only)
666 .value("cache_remove_from_disk", ioflags_cache_remove_from_disk)
669 enum_<elliptics_log_level>("log_level")
670 .value("data", log_level_data)
671 .value("error", log_level_error)
672 .value("info", log_level_info)
673 .value("notice", log_level_notice)
674 .value("debug", log_level_debug)