Merge branch 'master' of http://toshik-nb-w7.ld.yandex.ru/gitweb/elliptics into toshik
[elliptics.git] / bindings / python / elliptics_python.cpp
blobcf3ec80b3c5bd7480ac8fe94d68706c4f49a39be
1 /*
2 * 2008+ Copyright (c) Evgeniy Polyakov <zbr@ioremap.net>
3 * All rights reserved.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
16 #include <netdb.h>
17 #include <boost/python.hpp>
18 #include <boost/python/list.hpp>
19 #include <boost/python/dict.hpp>
21 #include <elliptics/cppdef.h>
23 using namespace boost::python;
24 using namespace ioremap::elliptics;
26 enum elliptics_cflags {
27 cflags_default = 0,
28 cflags_direct = DNET_FLAGS_DIRECT,
29 cflags_nolock = DNET_FLAGS_NOLOCK,
32 enum elliptics_ioflags {
33 ioflags_default = 0,
34 ioflags_append = DNET_IO_FLAGS_APPEND,
35 ioflags_compress = DNET_IO_FLAGS_COMPRESS,
36 ioflags_meta = DNET_IO_FLAGS_META,
37 ioflags_prepare = DNET_IO_FLAGS_PREPARE,
38 ioflags_commit = DNET_IO_FLAGS_COMMIT,
39 ioflags_overwrite = DNET_IO_FLAGS_OVERWRITE,
40 ioflags_nocsum = DNET_IO_FLAGS_NOCSUM,
41 ioflags_plain_write = DNET_IO_FLAGS_PLAIN_WRITE,
42 ioflags_cache = DNET_IO_FLAGS_CACHE,
43 ioflags_cache_only = DNET_IO_FLAGS_CACHE_ONLY,
44 ioflags_cache_remove_from_disk = DNET_IO_FLAGS_CACHE_REMOVE_FROM_DISK,
47 enum elliptics_log_level {
48 log_level_data = DNET_LOG_DATA,
49 log_level_error = DNET_LOG_ERROR,
50 log_level_info = DNET_LOG_INFO,
51 log_level_notice = DNET_LOG_NOTICE,
52 log_level_debug = DNET_LOG_DEBUG,
55 static void elliptics_extract_arr(const list &l, unsigned char *dst, int *dlen)
57 int length = len(l);
59 if (length > *dlen)
60 length = *dlen;
62 memset(dst, 0, *dlen);
63 for (int i = 0; i < length; ++i)
64 dst[i] = extract<unsigned char>(l[i]);
67 struct elliptics_id {
68 elliptics_id() : group_id(0), type(0) {}
69 elliptics_id(list id_, int group_, int type_) : id(id_), group_id(group_), type(type_) {}
71 elliptics_id(struct dnet_id &dnet) {
72 for (unsigned int i = 0; i < sizeof(dnet.id); ++i)
73 id.append(dnet.id[i]);
75 group_id = dnet.group_id;
76 type = dnet.type;
79 struct dnet_id to_dnet() const {
80 struct dnet_id dnet;
81 int len = sizeof(dnet.id);
83 elliptics_extract_arr(id, dnet.id, &len);
85 dnet.group_id = group_id;
86 dnet.type = type;
88 return dnet;
91 list id;
92 uint32_t group_id;
93 int type;
96 struct elliptics_range {
97 elliptics_range() : offset(0), size(0),
98 limit_start(0), limit_num(0), cflags(0), ioflags(0), group_id(0), type(0) {}
100 list start, end;
101 uint64_t offset, size;
102 uint64_t limit_start, limit_num;
103 uint64_t cflags;
104 uint32_t ioflags;
105 int group_id;
106 int type;
109 static void elliptics_extract_range(const struct elliptics_range &r, struct dnet_io_attr &io)
111 int len = sizeof(io.id);
113 elliptics_extract_arr(r.start, io.id, &len);
114 elliptics_extract_arr(r.end, io.parent, &len);
116 io.flags = r.ioflags;
117 io.size = r.size;
118 io.offset = r.offset;
119 io.start = r.limit_start;
120 io.num = r.limit_num;
121 io.type = r.type;
124 class elliptics_log_wrap : public logger, public wrapper<logger> {
125 public:
126 elliptics_log_wrap(const int level = DNET_LOG_INFO) : logger(level) {};
128 void log(const int level, const char *msg) {
129 this->get_override("log")(level, msg);
132 unsigned long clone(void) {
133 return this->get_override("clone")();
137 class elliptics_log_file_wrap : public log_file, public wrapper<log_file> {
138 public:
139 elliptics_log_file_wrap(const char *file, const int level = DNET_LOG_INFO) :
140 log_file(file, level) {};
142 void log(const int level, const char *msg) {
143 if (override log = this->get_override("log")) {
144 log_file::log(level, msg);
145 return;
148 log_file::log(level, msg);
151 void default_log(const int level, const char *msg) { this->log(level, msg); }
153 unsigned long clone(void) {
154 if (override clone = this->get_override("clone"))
155 return log_file::clone();
157 return log_file::clone();
160 unsigned long default_clone(void) { return this->clone(); }
163 class elliptics_config {
164 public:
165 elliptics_config() {
166 memset(&config, 0, sizeof(struct dnet_config));
169 std::string cookie_get(void) const {
170 std::string ret;
171 ret.assign(config.cookie, sizeof(config.cookie));
172 return ret;
175 void cookie_set(const std::string &cookie) {
176 size_t sz = sizeof(config.cookie);
177 if (cookie.size() + 1 < sz)
178 sz = cookie.size() + 1;
179 memset(config.cookie, 0, sizeof(config.cookie));
180 snprintf(config.cookie, sz, "%s", (char *)cookie.data());
183 struct dnet_config config;
186 class elliptics_node_python : public node, public wrapper<node> {
187 public:
188 elliptics_node_python(logger &l) : node(l) {}
190 elliptics_node_python(logger &l, elliptics_config &cfg) : node(l, cfg.config) {}
192 elliptics_node_python(const node &n): node(n) {}
197 class elliptics_session: public session, public wrapper<session> {
198 public:
199 elliptics_session(node &n) : session(n) {}
201 elliptics_session(const session &s): session(s) {}
203 void add_groups(const list &pgroups) {
204 std::vector<int> groups;
206 for (int i=0; i<len(pgroups); ++i)
207 groups.push_back(extract<int>(pgroups[i]));
209 session::add_groups(groups);
212 boost::python::list get_groups() {
213 std::vector<int> groups = session::get_groups();
214 boost::python::list res;
215 for(size_t i=0; i<groups.size(); i++) {
216 res.append(groups[i]);
219 return res;
222 void write_metadata_by_id(const struct elliptics_id &id, const std::string &remote, const list &pgroups, uint64_t cflags) {
223 struct timespec ts;
224 memset(&ts, 0, sizeof(ts));
226 struct dnet_id raw = id.to_dnet();
228 std::vector<int> groups;
230 for (int i=0; i<len(pgroups); ++i)
231 groups.push_back(extract<int>(pgroups[i]));
233 write_metadata((const dnet_id&)raw, remote, groups, ts, cflags);
236 void write_metadata_by_data_transform(const std::string &remote, uint64_t cflags) {
237 struct timespec ts;
238 memset(&ts, 0, sizeof(ts));
240 struct dnet_id raw;
242 transform(remote, raw);
244 write_metadata((const dnet_id&)raw, remote, groups, ts, cflags);
247 void read_file_by_id(struct elliptics_id &id, const std::string &file, uint64_t offset, uint64_t size) {
248 struct dnet_id raw = id.to_dnet();
249 read_file(raw, file, offset, size);
252 void read_file_by_data_transform(const std::string &remote, const std::string &file,
253 uint64_t offset, uint64_t size, int type) {
254 read_file(remote, file, offset, size, type);
257 void write_file_by_id(struct elliptics_id &id, const std::string &file,
258 uint64_t local_offset, uint64_t offset, uint64_t size,
259 uint64_t cflags, unsigned int ioflags) {
260 struct dnet_id raw = id.to_dnet();
261 write_file(raw, file, local_offset, offset, size, cflags, ioflags);
264 void write_file_by_data_transform(const std::string &remote, const std::string &file,
265 uint64_t local_offset, uint64_t offset, uint64_t size,
266 uint64_t cflags, unsigned int ioflags, int type) {
267 write_file(remote, file, local_offset, offset, size, cflags, ioflags, type);
270 std::string read_data_by_id(const struct elliptics_id &id, uint64_t offset, uint64_t size,
271 uint64_t cflags, unsigned int ioflags) {
272 struct dnet_id raw = id.to_dnet();
273 return read_data_wait(raw, offset, size, cflags, ioflags);
276 std::string read_data_by_data_transform(const std::string &remote, uint64_t offset, uint64_t size,
277 uint64_t cflags, unsigned int ioflags, int type) {
278 return read_data_wait(remote, offset, size, cflags, ioflags, type);
281 list prepare_latest_by_id(const struct elliptics_id &id, uint64_t cflags, list gl) {
282 struct dnet_id raw = id.to_dnet();
284 std::vector<int> groups;
285 for (int i = 0; i < len(gl); ++i)
286 groups.push_back(extract<int>(gl[i]));
288 prepare_latest(raw, cflags, groups);
290 list l;
291 for (unsigned i = 0; i < groups.size(); ++i)
292 l.append(groups[i]);
294 return l;
297 std::string prepare_latest_by_id_str(const struct elliptics_id &id, uint64_t cflags, list gl) {
298 struct dnet_id raw = id.to_dnet();
300 std::vector<int> groups;
301 for (int i = 0; i < len(gl); ++i)
302 groups.push_back(extract<int>(gl[i]));
304 prepare_latest(raw, cflags, groups);
306 std::string ret;
307 ret.assign((char *)groups.data(), groups.size() * 4);
309 return ret;
312 std::string read_latest_by_id(const struct elliptics_id &id, uint64_t offset, uint64_t size,
313 uint64_t cflags, unsigned int ioflags) {
314 struct dnet_id raw = id.to_dnet();
315 return read_latest(raw, offset, size, cflags, ioflags);
318 std::string read_latest_by_data_transform(const std::string &remote, uint64_t offset, uint64_t size,
319 uint64_t cflags, unsigned int ioflags, int type) {
320 return read_latest(remote, offset, size, cflags, ioflags, type);
323 std::string write_data_by_id(const struct elliptics_id &id, const std::string &data, uint64_t remote_offset,
324 uint64_t cflags, unsigned int ioflags) {
325 struct dnet_id raw = id.to_dnet();
326 return write_data_wait(raw, data, remote_offset, cflags, ioflags);
329 std::string write_data_by_data_transform(const std::string &remote, const std::string &data, uint64_t remote_offset,
330 uint64_t cflags, unsigned int ioflags, int type) {
331 return write_data_wait(remote, data, remote_offset, cflags, ioflags, type);
334 std::string write_cache_by_id(const struct elliptics_id &id, const std::string &data,
335 uint64_t cflags, unsigned int ioflags, long timeout) {
336 struct dnet_id raw = id.to_dnet();
337 raw.type = 0;
338 return write_cache(raw, data, cflags, ioflags, timeout);
341 std::string write_cache_by_data_transform(const std::string &remote, const std::string &data,
342 uint64_t cflags, unsigned int ioflags, long timeout) {
343 return write_cache(remote, data, cflags, ioflags, timeout);
346 std::string lookup_addr_by_data_transform(const std::string &remote, const int group_id) {
347 return lookup_addr(remote, group_id);
350 std::string lookup_addr_by_id(const struct elliptics_id &id) {
351 struct dnet_id raw = id.to_dnet();
353 return lookup_addr(raw);
356 boost::python::tuple parse_lookup(const std::string &lookup) {
357 const void *data = lookup.data();
359 struct dnet_addr *addr = (struct dnet_addr *)data;
360 struct dnet_cmd *cmd = (struct dnet_cmd *)(addr + 1);
361 struct dnet_addr_attr *a = (struct dnet_addr_attr *)(cmd + 1);
362 struct dnet_file_info *info = (struct dnet_file_info *)(a + 1);
363 dnet_convert_file_info(info);
365 std::string address(dnet_server_convert_dnet_addr(addr));
366 int port = dnet_server_convert_port((struct sockaddr *)a->addr.addr, a->addr.addr_len);
368 return make_tuple(address, port, info->size);
371 boost::python::tuple lookup_by_data_transform(const std::string &remote) {
372 return parse_lookup(lookup(remote));
375 boost::python::tuple lookup_by_id(const struct elliptics_id &id) {
376 struct dnet_id raw = id.to_dnet();
378 return parse_lookup(lookup(raw));
381 struct dnet_node_status update_status_by_id(const struct elliptics_id &id, struct dnet_node_status &status) {
382 struct dnet_id raw = id.to_dnet();
384 update_status(raw, &status);
385 return status;
388 struct dnet_node_status update_status_by_string(const std::string &saddr, const int port, const int family,
389 struct dnet_node_status &status) {
390 update_status(saddr.c_str(), port, family, &status);
391 return status;
394 boost::python::list read_data_range(const struct elliptics_range &r) {
395 struct dnet_io_attr io;
396 elliptics_extract_range(r, io);
398 std::vector<std::string> ret;
399 ret = session::read_data_range(io, r.group_id, r.cflags);
401 boost::python::list l;
403 for (size_t i = 0; i < ret.size(); ++i) {
404 l.append(ret[i]);
407 return l;
410 boost::python::list get_routes() {
412 std::vector<std::pair<struct dnet_id, struct dnet_addr> > routes;
413 std::vector<std::pair<struct dnet_id, struct dnet_addr> >::iterator it;
415 boost::python::list res;
417 routes = session::get_routes();
419 for (it = routes.begin(); it != routes.end(); it++) {
420 struct elliptics_id id(it->first);
421 std::string address(dnet_server_convert_dnet_addr(&(it->second)));
423 res.append(make_tuple(id, address));
426 return res;
429 std::string exec_name(const struct elliptics_id &id, const std::string &event,
430 const std::string &data, const std::string &binary) {
431 struct dnet_id raw = id.to_dnet();
433 return exec_locked(&raw, event, data, binary);
436 std::string exec_name_by_name(const std::string &remote, const std::string &event,
437 const std::string &data, const std::string &binary) {
438 struct dnet_id raw;
439 transform(remote, raw);
440 raw.type = 0;
441 raw.group_id = 0;
443 return exec_locked(&raw, event, data, binary);
446 std::string exec_name_all(const std::string &event, const std::string &data, const std::string &binary) {
447 return exec_locked(NULL, event, data, binary);
450 void remove_by_id(const struct elliptics_id &id, uint64_t cflags, uint64_t ioflags) {
451 struct dnet_id raw = id.to_dnet();
453 remove_raw(raw, cflags, ioflags);
456 void remove_by_name(const std::string &remote, uint64_t cflags, uint64_t ioflags, int type) {
457 remove_raw(remote, cflags, ioflags, type);
460 list bulk_read_by_name(const list &keys, uint64_t cflags = 0) {
461 unsigned int length = len(keys);
463 std::vector<std::string> k;
464 k.resize(length);
466 for (unsigned int i = 0; i < length; ++i)
467 k[i] = extract<std::string>(keys[i]);
469 std::vector<std::string> ret = bulk_read(k, cflags);
471 list py_ret;
472 for (size_t i = 0; i < ret.size(); ++i) {
473 py_ret.append(ret[i]);
476 return py_ret;
479 list stat_log() {
480 list statistics;
481 callback c;
482 std::string ret;
483 int err;
484 int i;
486 err = dnet_request_stat(m_session, NULL, DNET_CMD_STAT_COUNT, DNET_ATTR_CNTR_GLOBAL,
487 callback::complete_callback, (void *)&c);
488 if (err < 0) {
489 std::ostringstream str;
490 str << "Failed to request statistics: " << err;
491 throw std::runtime_error(str.str());
494 ret = c.wait(err);
496 const void *data = ret.data();
497 int size = ret.size();
499 while (size > 0) {
500 dict node_stat, storage_commands, proxy_commands, counters;
501 struct dnet_addr *addr = (struct dnet_addr *)data;
502 struct dnet_cmd *cmd = (struct dnet_cmd *)(addr + 1);
503 if (cmd->size <= sizeof(struct dnet_addr_stat)) {
504 size -= cmd->size + sizeof(struct dnet_addr) + sizeof(struct dnet_cmd);
505 data = (char *)data + cmd->size + sizeof(struct dnet_addr) + sizeof(struct dnet_cmd);
506 continue;
509 struct dnet_addr_stat *as = (struct dnet_addr_stat *)(cmd + 1);
511 dnet_convert_addr_stat(as, 0);
512 std::string address(dnet_server_convert_dnet_addr(addr));
513 node_stat[std::string("addr")] = address;
514 node_stat[std::string("group_id")] = cmd->id.group_id;
516 for (i = 0; i < as->num; ++i) {
517 if (i < as->cmd_num) {
518 storage_commands[std::string(dnet_counter_string(i, as->cmd_num))] =
519 make_tuple((unsigned long long)as->count[i].count, (unsigned long long)as->count[i].err);
520 } else if (i < (as->cmd_num * 2)) {
521 proxy_commands[std::string(dnet_counter_string(i, as->cmd_num))] =
522 make_tuple((unsigned long long)as->count[i].count, (unsigned long long)as->count[i].err);
523 } else {
524 counters[std::string(dnet_counter_string(i, as->cmd_num))] =
525 make_tuple((unsigned long long)as->count[i].count, (unsigned long long)as->count[i].err);
529 node_stat["storage_commands"] = storage_commands;
530 node_stat["proxy_commands"] = proxy_commands;
531 node_stat["counters"] = counters;
533 statistics.append(node_stat);
535 int sz = sizeof(struct dnet_addr) + sizeof(struct dnet_cmd) + cmd->size;
536 size -= sz;
537 data = (char *)data + sz;
540 return statistics;
543 BOOST_PYTHON_MEMBER_FUNCTION_OVERLOADS(add_remote_overloads, add_remote, 2, 3);
545 BOOST_PYTHON_MODULE(libelliptics_python) {
546 class_<elliptics_id>("elliptics_id", init<>())
547 .def(init<list, int, int>())
548 .def_readwrite("id", &elliptics_id::id)
549 .def_readwrite("group_id", &elliptics_id::group_id)
550 .def_readwrite("type", &elliptics_id::type)
553 class_<elliptics_range>("elliptics_range", init<>())
554 .def_readwrite("start", &elliptics_range::start)
555 .def_readwrite("end", &elliptics_range::end)
556 .def_readwrite("offset", &elliptics_range::offset)
557 .def_readwrite("size", &elliptics_range::size)
558 .def_readwrite("ioflags", &elliptics_range::ioflags)
559 .def_readwrite("cflags", &elliptics_range::cflags)
560 .def_readwrite("group_id", &elliptics_range::group_id)
561 .def_readwrite("type", &elliptics_range::type)
562 .def_readwrite("limit_start", &elliptics_range::limit_start)
563 .def_readwrite("limit_num", &elliptics_range::limit_num)
566 class_<elliptics_log_wrap, boost::noncopyable>("elliptics_log", init<const uint32_t>())
567 .def("log", pure_virtual(&logger::log))
568 .def("clone", pure_virtual(&logger::clone))
571 class_<elliptics_log_file_wrap, boost::noncopyable, bases<logger> >("elliptics_log_file", init<const char *, const uint32_t>())
572 .def("log", &log_file::log, &elliptics_log_file_wrap::default_log)
573 .def("clone", &log_file::clone, &elliptics_log_file_wrap::default_clone)
576 class_<dnet_node_status>("dnet_node_status", init<>())
577 .def_readwrite("nflags", &dnet_node_status::nflags)
578 .def_readwrite("status_flags", &dnet_node_status::status_flags)
579 .def_readwrite("log_level", &dnet_node_status::log_level)
582 class_<dnet_config>("dnet_config", init<>())
583 .def_readwrite("wait_timeout", &dnet_config::wait_timeout)
584 .def_readwrite("flags", &dnet_config::flags)
585 .def_readwrite("check_timeout", &dnet_config::check_timeout)
586 .def_readwrite("io_thread_num", &dnet_config::io_thread_num)
587 .def_readwrite("nonblocking_io_thread_num", &dnet_config::nonblocking_io_thread_num)
588 .def_readwrite("net_thread_num", &dnet_config::net_thread_num)
589 .def_readwrite("client_prio", &dnet_config::client_prio)
592 class_<elliptics_config>("elliptics_config", init<>())
593 .def_readwrite("config", &elliptics_config::config)
594 .add_property("cookie", &elliptics_config::cookie_get, &elliptics_config::cookie_set)
597 class_<elliptics_node_python>("elliptics_node_python", init<logger &>())
598 .def(init<logger &, elliptics_config &>())
599 .def("add_remote", &node::add_remote, add_remote_overloads())
602 class_<elliptics_session>("elliptics_session", init<node &>())
603 .def("add_groups", &elliptics_session::add_groups)
604 .def("get_groups", &elliptics_session::get_groups)
606 .def("read_file", &elliptics_session::read_file_by_id)
607 .def("read_file", &elliptics_session::read_file_by_data_transform)
608 .def("write_file", &elliptics_session::write_file_by_id)
609 .def("write_file", &elliptics_session::write_file_by_data_transform)
611 .def("read_data", &elliptics_session::read_data_by_id)
612 .def("read_data", &elliptics_session::read_data_by_data_transform)
614 .def("prepare_latest", &elliptics_session::prepare_latest_by_id)
615 .def("prepare_latest_str", &elliptics_session::prepare_latest_by_id_str)
617 .def("read_latest", &elliptics_session::read_latest_by_id)
618 .def("read_latest", &elliptics_session::read_latest_by_data_transform)
620 .def("write_data", &elliptics_session::write_data_by_id)
621 .def("write_data", &elliptics_session::write_data_by_data_transform)
623 .def("write_metadata", &elliptics_session::write_metadata_by_id)
624 .def("write_metadata", &elliptics_session::write_metadata_by_data_transform)
626 .def("write_cache", &elliptics_session::write_cache_by_id)
627 .def("write_cache", &elliptics_session::write_cache_by_data_transform)
629 .def("lookup_addr", &elliptics_session::lookup_addr_by_data_transform)
630 .def("lookup_addr", &elliptics_session::lookup_addr_by_id)
632 .def("lookup", &elliptics_session::lookup_by_data_transform)
633 .def("lookup", &elliptics_session::lookup_by_id)
635 .def("update_status", &elliptics_session::update_status_by_id)
636 .def("update_status", &elliptics_session::update_status_by_string)
638 .def("read_data_range", &elliptics_session::read_data_range)
640 .def("get_routes", &elliptics_session::get_routes)
641 .def("stat_log", &elliptics_session::stat_log)
643 .def("exec_event", &elliptics_session::exec_name)
644 .def("exec_event", &elliptics_session::exec_name_by_name)
645 .def("exec_event", &elliptics_session::exec_name_all)
647 .def("remove", &elliptics_session::remove_by_id)
648 .def("remove", &elliptics_session::remove_by_name)
650 .def("bulk_read", &elliptics_session::bulk_read_by_name)
653 enum_<elliptics_cflags>("command_flags")
654 .value("default", cflags_default)
655 .value("direct", cflags_direct)
656 .value("nolock", cflags_nolock)
659 enum_<elliptics_ioflags>("io_flags")
660 .value("default", ioflags_default)
661 .value("append", ioflags_append)
662 .value("compress", ioflags_compress)
663 .value("meta", ioflags_meta)
664 .value("prepare", ioflags_prepare)
665 .value("commit", ioflags_commit)
666 .value("overwrite", ioflags_overwrite)
667 .value("nocsum", ioflags_nocsum)
668 .value("plain_write", ioflags_plain_write)
669 .value("nodata", ioflags_plain_write)
670 .value("cache", ioflags_cache)
671 .value("cache_only", ioflags_cache_only)
672 .value("cache_remove_from_disk", ioflags_cache_remove_from_disk)
675 enum_<elliptics_log_level>("log_level")
676 .value("data", log_level_data)
677 .value("error", log_level_error)
678 .value("info", log_level_info)
679 .value("notice", log_level_notice)
680 .value("debug", log_level_debug)