src/watch/watch_mongodb.cpp

   1 #include <cstdlib>
   2 #include <iostream>
   3 #include <sstream>
   4 #include <limits>
   5 #include <stdexcept>
   6 #include <set>
   7 #include <map>
   8 #include <vector>
   9
  10 #include <client/dbclient.h>
  11
  12 #include <boost/lexical_cast.hpp>
  13
  14 #include <boost/program_options/options_description.hpp>
  15 #include <boost/program_options/parsers.hpp>
  16 #include <boost/program_options/variables_map.hpp>
  17 #include <boost/timer/timer.hpp>
  18 #include <boost/locale.hpp>
  19
  20 #include <algorithm>
  21 #include <boost/lambda/lambda.hpp>
  22
  23 #include "asn1.h"
  24
  25 using namespace mongo;
  26 using namespace std;
  27 using namespace boost;
  28 using namespace boost::program_options;
  29 using namespace boost::lambda;
  30
  31 #if 1
  32 #define DBNAME "admin"
  33 #else
  34 #define DBNAME "local"
  35 #endif
  36
  37 extern
  38 void connect(DBClientConnection &c, std::string const &dsn, std::string const &dbname, std::string const &user);
  39
  40 namespace boost
  41 {
  42
  43 template<>
  44 std::string
  45 lexical_cast<std::string>( bool const &v )
  46 {
  47     return v ? "true" : "false";
  48 }
  49
  50 }
  51
  52 string
  53 join( string const &delim, vector<string> const &list )
  54 {
  55     string rc;
  56
  57     for( vector<string>::const_iterator ci = list.begin(); ci != list.end(); ++ci )
  58     {
  59         if( !rc.empty() )
  60             rc += delim;
  61         rc += *ci;
  62     }
  63
  64     return rc;
  65 }
  66
  67 #define EXCEED_TYPE_BOUNDS(TYPE, VAL) \
  68         std::numeric_limits<TYPE>::is_integer && \
  69         ((VAL > std::numeric_limits<TYPE>::max()) || \
  70          (VAL < std::numeric_limits<TYPE>::min()))
  71
  72 template<class T>
  73 T
  74 extract_number(BSONElement const &e, string const &oid)
  75 {
  76     T result;
  77
  78     switch( e.type() )
  79     {
  80         case NumberDouble:
  81         {
  82             double tmp = e.Double();
  83             if( EXCEED_TYPE_BOUNDS(T, tmp) )
  84                 throw out_of_range(
  85                     string("double value for ") + oid +
  86                     " (" + lexical_cast<string>(tmp) + ") not between " +
  87                     lexical_cast<string>(std::numeric_limits<T>::min()) +
  88                     " and " +
  89                     lexical_cast<string>(std::numeric_limits<T>::max()));
  90             return (T)tmp;
  91         }
  92         case Bool:
  93         {
  94             int tmp = e.Bool();
  95             return (T)tmp;
  96         }
  97         case NumberInt:
  98         {
  99             long long tmp = e.Int();
 100             if( EXCEED_TYPE_BOUNDS(T, tmp) )
 101                 throw out_of_range(
 102                     string("int value for ") + oid +
 103                     " (" + lexical_cast<string>(tmp) + ") not between " +
 104                     lexical_cast<string>(std::numeric_limits<T>::min()) +
 105                     " and " +
 106                     lexical_cast<string>(std::numeric_limits<T>::max()));
 107             return (T)tmp;
 108         }
 109         case Date:
 110         {
 111             unsigned long long tmp = e.Date();
 112             if( EXCEED_TYPE_BOUNDS(T, tmp) )
 113                 throw out_of_range(
 114                     string("int value for ") + oid +
 115                     " (" + lexical_cast<string>(tmp) + ") not between " +
 116                     lexical_cast<string>(std::numeric_limits<T>::min()) +
 117                     " and " +
 118                     lexical_cast<string>(std::numeric_limits<T>::max()));
 119             return (T)tmp;
 120         }
 121         case Timestamp:
 122         {
 123             if( std::numeric_limits<T>::max() <= std::numeric_limits<unsigned int>::max() )
 124             {
 125                 unsigned int tmp = e.timestampInc();
 126                 if( EXCEED_TYPE_BOUNDS(T, tmp) )
 127                     throw out_of_range(
 128                         string("int value for ") + oid +
 129                         " (" + lexical_cast<string>(tmp) + ") not between " +
 130                         lexical_cast<string>(std::numeric_limits<T>::min()) +
 131                         " and " +
 132                         lexical_cast<string>(std::numeric_limits<T>::max()));
 133                 return (T)tmp;
 134             }
 135             else
 136             {
 137                 unsigned long long tmp = e.timestampTime();
 138                 if( EXCEED_TYPE_BOUNDS(T, tmp) )
 139                     throw out_of_range(
 140                         string("int value for ") + oid +
 141                         " (" + lexical_cast<string>(tmp) + ") not between " +
 142                         lexical_cast<string>(std::numeric_limits<T>::min()) +
 143                         " and " +
 144                         lexical_cast<string>(std::numeric_limits<T>::max()));
 145                 return (T)tmp;
 146             }
 147         }
 148         case NumberLong:
 149         {
 150             long long tmp = e.Long();
 151             if( EXCEED_TYPE_BOUNDS(T, tmp) )
 152                 throw out_of_range(
 153                     string("long long value for ") + oid +
 154                     " (" + lexical_cast<string>(tmp) + ") not between " +
 155                     lexical_cast<string>(std::numeric_limits<T>::min()) +
 156                     " and " +
 157                     lexical_cast<string>(std::numeric_limits<T>::max()));
 158             return (T)tmp;
 159         }
 160         default:
 161         {
 162             StringBuilder ss;
 163             ss << "wrong type for field (" << e.fieldName() << ") " << e.type() << " not in "
 164                << NumberDouble << ", "
 165                << Bool << ", "
 166                << NumberInt << ", "
 167                << Timestamp << " or "
 168                << NumberLong;
 169             msgasserted(13111, ss.str() );
 170             break;
 171         }
 172     }
 173 }
 174
 175 struct OidValueTuple
 176 {
 177     string oid;
 178     unsigned type;
 179     string value;
 180
 181     OidValueTuple(string const &an_oid, unsigned a_type = ASN_NULL, string const &a_value = string())
 182         : oid(an_oid)
 183         , type(a_type)
 184         , value(a_value)
 185     {}
 186 };
 187
 188 inline bool
 189 operator < (OidValueTuple const &x, OidValueTuple const &y)
 190 {
 191     return x.oid < y.oid;
 192 }
 193
 194 template<class T>
 195 OidValueTuple
 196 extract(BSONElement const &e, string const &oid)
 197 {
 198     return OidValueTuple( oid );
 199 }
 200
 201 template<>
 202 OidValueTuple
 203 extract<string>(BSONElement const &e, string const &oid)
 204 {
 205     return OidValueTuple( oid, ASN_OCTET_STR, e.String() );
 206 }
 207
 208 template<>
 209 OidValueTuple
 210 extract<int>(BSONElement const &e, string const &oid)
 211 {
 212     return OidValueTuple( oid, ASN_INTEGER, lexical_cast<string>( extract_number<int>(e, oid) ) );
 213 }
 214
 215 template<>
 216 OidValueTuple
 217 extract<unsigned int>(BSONElement const &e, string const &oid)
 218 {
 219     return OidValueTuple( oid, SMI_UINTEGER, lexical_cast<string>( extract_number<unsigned int>(e, oid) ) );
 220 }
 221
 222 template<>
 223 OidValueTuple
 224 extract<unsigned long long>(BSONElement const &e, string const &oid)
 225 {
 226     return OidValueTuple( oid, SMI_COUNTER64, lexical_cast<string>( extract_number<unsigned long long>(e, oid) ) );
 227 }
 228
 229 template<>
 230 OidValueTuple
 231 extract<double>(BSONElement const &e, string const &oid)
 232 {
 233     return OidValueTuple( oid, ASN_OCTET_STR, lexical_cast<string>( extract_number<double>(e, oid) ) );
 234 }
 235
 236 struct Extractor
 237 {
 238 public:
 239     virtual void operator()(BSONElement const &e, set<OidValueTuple> &out_vals) = 0;
 240 };
 241
 242 ostream &
 243 operator << (ostream &os, OidValueTuple const val)
 244 {
 245     os << "(" << val.oid << ", " << val.type << ", " << val.value << ")";
 246     return os;
 247 }
 248
 249 template<class T>
 250 struct ItemExtractor
 251     : public Extractor
 252 {
 253 public:
 254     ItemExtractor(string const &oid)
 255         : Extractor()
 256         , m_oid(oid)
 257     {}
 258
 259     virtual void operator()(BSONElement const &e, set<OidValueTuple> &out_vals)
 260     {
 261         out_vals.insert( extract<T>( e, m_oid ) );
 262     }
 263
 264 protected:
 265     string const m_oid;
 266
 267 private:
 268     ItemExtractor();
 269 };
 270
 271 struct StructExtractor
 272     : public Extractor
 273 {
 274 public:
 275     StructExtractor(map<string, Extractor *> *item_rules)
 276         : Extractor()
 277         , m_item_rules(item_rules)
 278     {}
 279
 280     virtual void operator()(BSONElement const &e, set<OidValueTuple> &out_vals)
 281     {
 282         BSONObjIterator i(e.Obj());
 283         while( i.more() )
 284         {
 285             BSONElement elem = i.next();
 286             string fname = elem.fieldName();
 287
 288             map<string, Extractor *>::iterator iter = m_item_rules->find( fname );
 289             if( iter != m_item_rules->end() )
 290             {
 291                 Extractor &ex = *iter->second;
 292                 ex(elem, out_vals);
 293             }
 294         }
 295     }
 296
 297     virtual void operator()(BSONObj const &o, set<OidValueTuple> &out_vals)
 298     {
 299         BSONObjIterator i(o.begin());
 300         while( i.more() )
 301         {
 302             BSONElement elem = i.next();
 303             string fname = elem.fieldName();
 304
 305             map<string, Extractor *>::iterator iter = m_item_rules->find( fname );
 306             if( iter != m_item_rules->end() )
 307             {
 308                 Extractor &ex = *iter->second;
 309                 ex(elem, out_vals);
 310             }
 311         }
 312     }
 313
 314     virtual ~StructExtractor()
 315     {
 316         for( map<string, Extractor *>::iterator i = m_item_rules->begin();
 317              i != m_item_rules->end();
 318              ++i )
 319         {
 320             delete i->second;
 321             i->second = 0;
 322         }
 323
 324         delete m_item_rules;
 325     }
 326
 327 protected:
 328     map<string, Extractor *> *m_item_rules;
 329
 330 private:
 331     StructExtractor();
 332     StructExtractor(StructExtractor const &);
 333     StructExtractor & operator = (StructExtractor const &);
 334 };
 335
 336 struct Anyfix
 337 {
 338     virtual string operator()() const = 0;
 339 };
 340
 341 struct StaticAnyfix
 342     : public Anyfix
 343 {
 344     StaticAnyfix(string const &anyfix)
 345         : Anyfix()
 346         , m_anyfix(anyfix)
 347     {}
 348
 349     virtual string operator()() const { return m_anyfix; }
 350
 351 protected:
 352      string m_anyfix;
 353 };
 354
 355 #if 0
 356 template<class Key, class Compare = less<Key>, class Allocator = allocator<Key> >
 357 std::set<Key, Compare, Allocator>::iterator
 358 insert_or_update( std::set<Key, Compare, Allocator> &vals, Key const &v )
 359 {
 360     std::set<Key, Compare, Allocator>::iterator i = vals.lower_bound(v);
 361     if( ( i == vals.end() ) || ( vals.key_comp()(v, *i ) ) )
 362     {
 363         i = vals.insert( i, v );
 364     }
 365     else
 366     {
 367         Key &ev = const_cast<Key &>(*i);
 368         ev = v;
 369     }
 370
 371     return i;
 372 }
 373 #endif
 374
 375 template<class Embed, class Pre, class Post>
 376 struct BothfixStructExtractor
 377     : public Embed
 378 {
 379 public:
 380     BothfixStructExtractor(Pre prefix, Post postfix)
 381         : Embed()
 382         , m_prefix(prefix)
 383         , m_postfix(postfix)
 384     {}
 385
 386     virtual void operator()(BSONElement const &e, set<OidValueTuple> &out_vals)
 387     {
 388         set<OidValueTuple> ofs_vals;
 389         Embed::operator()(e, ofs_vals);
 390         apply_fixes(ofs_vals, out_vals);
 391     }
 392
 393     void apply_fixes(set<OidValueTuple> &collected_vals, set<OidValueTuple> &out_vals)
 394     {
 395         for( set<OidValueTuple>::iterator iter = collected_vals.begin();
 396              iter != collected_vals.end();
 397              ++iter )
 398         {
 399             OidValueTuple ov = *iter;
 400             ov.oid = m_prefix() + ov.oid + m_postfix();
 401             insert_or_update( out_vals, ov );
 402         }
 403     }
 404
 405 protected:
 406     Pre m_prefix;
 407     Post m_postfix;
 408
 409     std::set<OidValueTuple>::iterator
 410     insert_or_update( std::set<OidValueTuple> &vals, OidValueTuple const &v )
 411     {
 412         std::set<OidValueTuple>::iterator i = vals.lower_bound(v);
 413         if( ( i == vals.end() ) || ( vals.key_comp()(v, *i ) ) )
 414         {
 415             i = vals.insert( i, v );
 416         }
 417         else
 418         {
 419             OidValueTuple &ev = const_cast<OidValueTuple &>(*i);
 420             ev = v;
 421         }
 422
 423         return i;
 424     }
 425
 426 private:
 427     BothfixStructExtractor();
 428     BothfixStructExtractor(StructExtractor const &);
 429     BothfixStructExtractor & operator = (BothfixStructExtractor const &);
 430 };
 431
 432 struct RowPostfix
 433     : public Anyfix
 434 {
 435     friend void swap(RowPostfix &a, RowPostfix &b);
 436
 437 public:
 438     RowPostfix(unsigned row = 0)
 439         : Anyfix()
 440         , m_row(row)
 441     {}
 442
 443     virtual string operator()() const { return "." + lexical_cast<string>(m_row); }
 444
 445     operator unsigned() const { return m_row; }
 446
 447     unsigned getRow() const { return m_row; }
 448     unsigned nextRow() { return ++m_row; }
 449     unsigned setNextRow( unsigned next_row ) { return m_row = next_row; }
 450
 451 protected:
 452     unsigned m_row;
 453 };
 454
 455 void
 456 swap(RowPostfix &a, RowPostfix &b)
 457 {
 458     swap(a.m_row, b.m_row);
 459 }
 460
 461 template<class T>
 462 struct ServReplHostsExtractor
 463     : public ItemExtractor<T>
 464 {
 465 public:
 466     ServReplHostsExtractor(string const &type)
 467         : ItemExtractor<T>(".2")
 468         , m_type(type)
 469     {}
 470
 471     virtual void operator()(BSONElement const &e, set<OidValueTuple> &out_vals)
 472     {
 473         ItemExtractor<T>::operator()( e, out_vals );
 474
 475         out_vals.insert( OidValueTuple( ".5", ASN_OCTET_STR, m_type ) );
 476     }
 477
 478 protected:
 479     string m_type;
 480 };
 481
 482 struct ServReplWorkersExtractor
 483     : public ServReplHostsExtractor<string>
 484 {
 485     ServReplWorkersExtractor()
 486         : ServReplHostsExtractor<string>("PRIORSEC")
 487     {}
 488 };
 489
 490 struct ServReplArbiterExtractor
 491     : public ServReplHostsExtractor<string>
 492 {
 493     ServReplArbiterExtractor()
 494         : ServReplHostsExtractor<string>("ARBITER")
 495     {}
 496 };
 497
 498 template <class E>
 499 struct TableRowExtractor
 500     : public BothfixStructExtractor<E, const StaticAnyfix, RowPostfix &>
 501 {
 502 public:
 503     TableRowExtractor(string const &tblOid, RowPostfix &rowPostfix, vector<string> const &key_chk = vector<string>() )
 504         : BothfixStructExtractor<E, const StaticAnyfix, RowPostfix &>(StaticAnyfix(tblOid + ".1"), rowPostfix)
 505         , m_key_chk(key_chk)
 506     {}
 507
 508     virtual void operator()(BSONElement const &e, set<OidValueTuple> &out_vals)
 509     {
 510         unsigned merge_row;
 511         set<OidValueTuple> embed_vals;
 512
 513         E::operator()(e, embed_vals);
 514         if( ( merge_row = find_key(embed_vals, out_vals) ) > 0 )
 515         {
 516             RowPostfix save(merge_row);
 517
 518             swap(save, this->m_postfix);
 519             this->apply_fixes(embed_vals, out_vals);
 520             swap(save, this->m_postfix);
 521         }
 522         else
 523         {
 524             if( merge_row < 0 )
 525                 this->m_postfix.setNextRow( 0 - ( merge_row - 1 ) );
 526             else
 527                 this->m_postfix.nextRow();
 528             this->apply_fixes(embed_vals, out_vals);
 529         }
 530     }
 531
 532     unsigned find_key(set<OidValueTuple> const &embed_vals, set<OidValueTuple> &out_vals) const
 533     {
 534         vector<bool> found;
 535         for( vector<string>::const_iterator ci = m_key_chk.begin();
 536              ci != m_key_chk.end();
 537              ++ci )
 538         {
 539             OidValueTuple search_key( *ci, ASN_OCTET_STR );
 540             set<OidValueTuple>::iterator cmp_iter = embed_vals.lower_bound(search_key);
 541             if( cmp_iter == embed_vals.end() )
 542                 continue;
 543
 544             search_key.value = cmp_iter->value;
 545             search_key.oid = this->m_prefix() + search_key.oid + ".";
 546
 547             for( cmp_iter = out_vals.lower_bound(search_key);
 548                  ( cmp_iter != out_vals.end() ) && ( 0 == cmp_iter->oid.compare( 0, search_key.oid.length(), search_key.oid ) );
 549                  ++cmp_iter )
 550             {
 551                 if( cmp_iter->value == search_key.value )
 552                 {
 553                     string row_str = cmp_iter->oid.substr( cmp_iter->oid.find_last_of("." ) + 1 );
 554                     unsigned row = lexical_cast<unsigned>(row_str);
 555                     if( found.size() < (row+1) )
 556                         found.resize(row+1);
 557
 558                     if( ci == m_key_chk.begin() )
 559                         found[row] = true;
 560                     else
 561                         found[row] = found[row] & true;
 562                 }
 563             }
 564         }
 565
 566         // scan found for first full matching row
 567         unsigned row = 0;
 568         for( vector<bool>::iterator vbi = found.begin();
 569              vbi != found.end();
 570              ++vbi, ++row )
 571         {
 572             if( *vbi )
 573                 return row;
 574         }
 575
 576         return 0;
 577     }
 578
 579 protected:
 580     vector<string> m_key_chk;
 581
 582 private:
 583     TableRowExtractor();
 584 };
 585
 586 template<class E>
 587 struct ListRowExtractor
 588     : public TableRowExtractor<E>
 589 {
 590 public:
 591     ListRowExtractor( string const &tblOid, RowPostfix &rowPostfix = RowPostfix(), vector<string> const &key_chk = vector<string>() )
 592         : TableRowExtractor<E>( tblOid, rowPostfix, key_chk )
 593     {}
 594
 595     virtual void operator()(BSONElement const &e, set<OidValueTuple> &out_vals)
 596     {
 597         BSONObjIterator i(e.Obj());
 598         while( i.more() )
 599         {
 600             BSONElement elem = i.next();
 601             TableRowExtractor<E>::operator()(elem, out_vals);
 602         }
 603     }
 604 };
 605
 606 Extractor *
 607 global_lock_extractors()
 608 {
 609     map<string, Extractor *> *extractor_map = new map<string, Extractor *>;
 610
 611     extractor_map->insert( make_pair<string, Extractor *>( "totalTime", new ItemExtractor<unsigned long long>( ".10.1" ) ) );
 612     extractor_map->insert( make_pair<string, Extractor *>( "lockTime", new ItemExtractor<unsigned long long>( ".10.2" ) ) );
 613
 614     map<string, Extractor *> *current_queue_map = new map<string, Extractor *>;
 615     current_queue_map->insert( make_pair<string, Extractor *>( "total", new ItemExtractor<unsigned long long>( ".10.3.1" ) ) );
 616     current_queue_map->insert( make_pair<string, Extractor *>( "readers", new ItemExtractor<unsigned long long>( ".10.3.2" ) ) );
 617     current_queue_map->insert( make_pair<string, Extractor *>( "writers", new ItemExtractor<unsigned long long>( ".10.3.3" ) ) );
 618     extractor_map->insert( make_pair<string, Extractor *>( "lockTime", new StructExtractor(current_queue_map) ) );
 619
 620     map<string, Extractor *> *active_clients_map = new map<string, Extractor *>;
 621     active_clients_map->insert( make_pair<string, Extractor *>( "total", new ItemExtractor<unsigned long long>( ".10.4.1" ) ) );
 622     active_clients_map->insert( make_pair<string, Extractor *>( "readers", new ItemExtractor<unsigned long long>( ".10.4.2" ) ) );
 623     active_clients_map->insert( make_pair<string, Extractor *>( "writers", new ItemExtractor<unsigned long long>( ".10.4.3" ) ) );
 624     extractor_map->insert( make_pair<string, Extractor *>( "lockTime", new StructExtractor(active_clients_map) ) );
 625
 626     return new StructExtractor(extractor_map);
 627 }
 628
 629 Extractor *
 630 mem_extractors()
 631 {
 632     map<string, Extractor *> *extractor_map = new map<string, Extractor *>;
 633
 634     extractor_map->insert( make_pair<string, Extractor *>( "bits", new ItemExtractor<unsigned int>( ".11.1" ) ) );
 635     extractor_map->insert( make_pair<string, Extractor *>( "resident", new ItemExtractor<unsigned long long>( ".11.2" ) ) );
 636     extractor_map->insert( make_pair<string, Extractor *>( "virtual", new ItemExtractor<unsigned long long>( ".11.3" ) ) );
 637     extractor_map->insert( make_pair<string, Extractor *>( "supported", new ItemExtractor<unsigned long long>( ".11.4" ) ) );
 638     extractor_map->insert( make_pair<string, Extractor *>( "mapped", new ItemExtractor<unsigned long long>( ".11.5" ) ) );
 639
 640     return new StructExtractor(extractor_map);
 641 }
 642
 643 Extractor *
 644 connections_extractors()
 645 {
 646     map<string, Extractor *> *extractor_map = new map<string, Extractor *>;
 647
 648     extractor_map->insert( make_pair<string, Extractor *>( "current", new ItemExtractor<unsigned int>( ".12.1" ) ) );
 649     extractor_map->insert( make_pair<string, Extractor *>( "available", new ItemExtractor<unsigned long long>( ".12.2" ) ) );
 650
 651     return new StructExtractor(extractor_map);
 652 }
 653
 654 Extractor *
 655 bg_flush_extractors()
 656 {
 657     map<string, Extractor *> *extractor_map = new map<string, Extractor *>;
 658
 659     extractor_map->insert( make_pair<string, Extractor *>( "flushes", new ItemExtractor<unsigned long long>( ".13.1" ) ) );
 660     extractor_map->insert( make_pair<string, Extractor *>( "total_ms", new ItemExtractor<unsigned long long>( ".13.2" ) ) );
 661     extractor_map->insert( make_pair<string, Extractor *>( "average_ms", new ItemExtractor<unsigned long long>( ".13.3" ) ) );
 662     extractor_map->insert( make_pair<string, Extractor *>( "last_ms", new ItemExtractor<unsigned long long>( ".13.4" ) ) );
 663     extractor_map->insert( make_pair<string, Extractor *>( "last_finished", new ItemExtractor<unsigned long long>( ".13.5" ) ) );
 664
 665     return new StructExtractor(extractor_map);
 666 }
 667
 668 Extractor *
 669 cursors_extractors()
 670 {
 671     map<string, Extractor *> *extractor_map = new map<string, Extractor *>;
 672
 673     extractor_map->insert( make_pair<string, Extractor *>( "totalOpen", new ItemExtractor<unsigned long long>( ".14.1" ) ) );
 674     extractor_map->insert( make_pair<string, Extractor *>( "clientCursors_size", new ItemExtractor<unsigned long long>( ".14.2" ) ) );
 675     extractor_map->insert( make_pair<string, Extractor *>( "timedOut", new ItemExtractor<unsigned long long>( ".14.3" ) ) );
 676
 677     return new StructExtractor(extractor_map);
 678 }
 679
 680 Extractor *
 681 network_extractors()
 682 {
 683     map<string, Extractor *> *extractor_map = new map<string, Extractor *>;
 684
 685     extractor_map->insert( make_pair<string, Extractor *>( "bytesIn", new ItemExtractor<unsigned long long>( ".15.1" ) ) );
 686     extractor_map->insert( make_pair<string, Extractor *>( "bytesOut", new ItemExtractor<unsigned long long>( ".15.2" ) ) );
 687     extractor_map->insert( make_pair<string, Extractor *>( "numRequests", new ItemExtractor<unsigned long long>( ".15.3" ) ) );
 688
 689     return new StructExtractor(extractor_map);
 690 }
 691
 692 Extractor *
 693 opcounters_extractors()
 694 {
 695     map<string, Extractor *> *extractor_map = new map<string, Extractor *>;
 696
 697     extractor_map->insert( make_pair<string, Extractor *>( "insert", new ItemExtractor<unsigned long long>( ".16.1" ) ) );
 698     extractor_map->insert( make_pair<string, Extractor *>( "query", new ItemExtractor<unsigned long long>( ".16.2" ) ) );
 699     extractor_map->insert( make_pair<string, Extractor *>( "update", new ItemExtractor<unsigned long long>( ".16.3" ) ) );
 700     extractor_map->insert( make_pair<string, Extractor *>( "delete", new ItemExtractor<unsigned long long>( ".16.4" ) ) );
 701     extractor_map->insert( make_pair<string, Extractor *>( "getmore", new ItemExtractor<unsigned long long>( ".16.5" ) ) );
 702     extractor_map->insert( make_pair<string, Extractor *>( "command", new ItemExtractor<unsigned long long>( ".16.6" ) ) );
 703
 704     return new StructExtractor(extractor_map);
 705 }
 706
 707 Extractor *
 708 asserts_extractors()
 709 {
 710     map<string, Extractor *> *extractor_map = new map<string, Extractor *>;
 711
 712     extractor_map->insert( make_pair<string, Extractor *>( "regular", new ItemExtractor<unsigned long long>( ".17.1" ) ) );
 713     extractor_map->insert( make_pair<string, Extractor *>( "warning", new ItemExtractor<unsigned long long>( ".17.2" ) ) );
 714     extractor_map->insert( make_pair<string, Extractor *>( "msg", new ItemExtractor<unsigned long long>( ".17.3" ) ) );
 715     extractor_map->insert( make_pair<string, Extractor *>( "user", new ItemExtractor<unsigned long long>( ".17.4" ) ) );
 716     extractor_map->insert( make_pair<string, Extractor *>( "rollovers", new ItemExtractor<unsigned long long>( ".17.5" ) ) );
 717
 718     return new StructExtractor(extractor_map);
 719 }
 720
 721 Extractor *
 722 record_stats_extractors()
 723 {
 724     map<string, Extractor *> *extractor_map = new map<string, Extractor *>;
 725
 726     extractor_map->insert( make_pair<string, Extractor *>( "accessesNotInMemory", new ItemExtractor<unsigned long long>( ".18.1" ) ) );
 727     extractor_map->insert( make_pair<string, Extractor *>( "pageFaultExceptionsThrown", new ItemExtractor<unsigned long long>( ".18.2" ) ) );
 728
 729     return new StructExtractor(extractor_map);
 730 }
 731
 732 struct LocksExtractor
 733     : public StructExtractor
 734 {
 735 public:
 736     LocksExtractor(int row)
 737         : StructExtractor(get_extractor_map(row))
 738     {}
 739
 740 protected:
 741     static map<string, Extractor *> *
 742     get_extractor_map(int row)
 743     {
 744         map<string, Extractor *> *overall_map = new map<string, Extractor *>;
 745
 746         if( 0 == row )
 747         {
 748             map<string, Extractor *> *details_map = new map<string, Extractor *>;
 749             details_map->insert( make_pair<string, Extractor *>( "R", new ItemExtractor<unsigned long long>( ".19.1.1" ) ) );
 750             details_map->insert( make_pair<string, Extractor *>( "W", new ItemExtractor<unsigned long long>( ".19.1.2" ) ) );
 751             overall_map->insert( make_pair<string, Extractor *>( "timeLockedMicros", new StructExtractor(details_map) ) );
 752
 753             details_map = new map<string, Extractor *>;
 754             details_map->insert( make_pair<string, Extractor *>( "R", new ItemExtractor<unsigned long long>( ".19.2.1" ) ) );
 755             details_map->insert( make_pair<string, Extractor *>( "W", new ItemExtractor<unsigned long long>( ".19.2.2" ) ) );
 756             overall_map->insert( make_pair<string, Extractor *>( "timeAcquiringMicros", new StructExtractor(details_map) ) );
 757         }
 758         else
 759         {
 760             map<string, Extractor *> *details_map = new map<string, Extractor *>;
 761             details_map->insert( make_pair<string, Extractor *>( "r", new ItemExtractor<unsigned long long>( ".21.1.14." + lexical_cast<string>(row) ) ) );
 762             details_map->insert( make_pair<string, Extractor *>( "w", new ItemExtractor<unsigned long long>( ".21.1.15." + lexical_cast<string>(row) ) ) );
 763             overall_map->insert( make_pair<string, Extractor *>( "timeLockedMicros", new StructExtractor(details_map) ) );
 764
 765             details_map = new map<string, Extractor *>;
 766             details_map->insert( make_pair<string, Extractor *>( "r", new ItemExtractor<unsigned long long>( ".21.1.16." + lexical_cast<string>(row) ) ) );
 767             details_map->insert( make_pair<string, Extractor *>( "w", new ItemExtractor<unsigned long long>( ".21.1.17." + lexical_cast<string>(row) ) ) );
 768             overall_map->insert( make_pair<string, Extractor *>( "timeAcquiringMicros", new StructExtractor(details_map) ) );
 769         }
 770
 771         return overall_map;
 772     }
 773 };
 774
 775
 776 Extractor *
 777 locks_extractors(vector<string> const &dbnames)
 778 {
 779     map<string, Extractor *> *extractor_map = new map<string, Extractor *>;
 780
 781     extractor_map->insert( make_pair<string, Extractor *>( ".", new LocksExtractor(0) ) );
 782     unsigned row = 1;
 783     for( vector<string>::const_iterator ci = dbnames.begin(); ci != dbnames.end(); ++ci, ++row )
 784         extractor_map->insert( make_pair<string, Extractor *>( *ci, new LocksExtractor(row) ) );
 785
 786     return new StructExtractor(extractor_map);
 787 }
 788
 789 StructExtractor *
 790 serv_info_repl_extractors()
 791 {
 792     map<string, Extractor *> *extractor_map = new map<string, Extractor *>;
 793
 794     extractor_map->insert( make_pair<string, Extractor *>( "setName", new ItemExtractor<string>( ".20.1" ) ) );
 795     extractor_map->insert( make_pair<string, Extractor *>( "ismaster", new ItemExtractor<int>( ".20.2" ) ) );
 796     extractor_map->insert( make_pair<string, Extractor *>( "secondary", new ItemExtractor<int>( ".20.3" ) ) );
 797     extractor_map->insert( make_pair<string, Extractor *>( "me", new ItemExtractor<string>( ".20.4" ) ) );
 798     extractor_map->insert( make_pair<string, Extractor *>( "primary", new ItemExtractor<string>( ".20.5" ) ) );
 799
 800     static RowPostfix repl_rows;
 801     vector<string> repl_tbl;
 802     repl_tbl.push_back(".2");
 803     extractor_map->insert( make_pair<string, Extractor *>(
 804         "hosts", new ListRowExtractor<ServReplWorkersExtractor>(".20.7", repl_rows, repl_tbl) ) );
 805     extractor_map->insert( make_pair<string, Extractor *>(
 806         "arbiters", new ListRowExtractor<ServReplArbiterExtractor>(".20.7", repl_rows, repl_tbl) ) );
 807
 808     return new StructExtractor(extractor_map);
 809 }
 810
 811 Extractor *
 812 repl_network_queue_extractors()
 813 {
 814     map<string, Extractor *> *extractor_map = new map<string, Extractor *>;
 815
 816     extractor_map->insert( make_pair<string, Extractor *>( "waitTimeMs", new ItemExtractor<unsigned long long>( ".20.6.1" ) ) );
 817     extractor_map->insert( make_pair<string, Extractor *>( "numElems", new ItemExtractor<unsigned long long>( ".20.6.2" ) ) );
 818     extractor_map->insert( make_pair<string, Extractor *>( "numBytes", new ItemExtractor<unsigned long long>( ".20.6.3" ) ) );
 819
 820     return new StructExtractor(extractor_map);
 821 }
 822
 823 StructExtractor *
 824 server_status_extractors(vector<string> const &dbnames)
 825 {
 826     map<string, Extractor *> *extractor_map = new map<string, Extractor *>;
 827
 828     extractor_map->insert( make_pair<string, Extractor *>( "host", new ItemExtractor<string>( ".1" ) ) );
 829     extractor_map->insert( make_pair<string, Extractor *>( "version", new ItemExtractor<string>( ".2" ) ) );
 830     extractor_map->insert( make_pair<string, Extractor *>( "process", new ItemExtractor<string>( ".3" ) ) );
 831     extractor_map->insert( make_pair<string, Extractor *>( "pid", new ItemExtractor<int>( ".4" ) ) );
 832     extractor_map->insert( make_pair<string, Extractor *>( "uptimeMillis", new ItemExtractor<unsigned long long>( ".5" ) ) );
 833
 834     extractor_map->insert( make_pair<string, Extractor *>( "globalLock", global_lock_extractors() ) );
 835     extractor_map->insert( make_pair<string, Extractor *>( "mem", mem_extractors() ) );
 836     extractor_map->insert( make_pair<string, Extractor *>( "connections", connections_extractors() ) );
 837     extractor_map->insert( make_pair<string, Extractor *>( "backgroundFlushing", bg_flush_extractors() ) );
 838     extractor_map->insert( make_pair<string, Extractor *>( "cursors", cursors_extractors() ) );
 839     extractor_map->insert( make_pair<string, Extractor *>( "network", network_extractors() ) );
 840     extractor_map->insert( make_pair<string, Extractor *>( "opcounters", opcounters_extractors() ) );
 841     extractor_map->insert( make_pair<string, Extractor *>( "asserts", asserts_extractors() ) );
 842     extractor_map->insert( make_pair<string, Extractor *>( "recordStats", record_stats_extractors() ) );
 843
 844 /*  XXX
 845                 if( serv_status.hasField("locks") && serv_status["locks"].Obj().hasField(dbname.value.c_str()) );
 846                 {
 847                     BSONObj dbl = serv_status["locks"].Obj().getField(dbname.value).Obj();
 848
 849                     if( dbl.hasField("timeLockedMicros") )
 850                     {
 851                         BSONObj o3 = dbl["timeLockedMicros"].Obj();
 852
 853                         if( o3.hasField("r") )
 854                             out_vals.insert( extract_uint64( o3["r"], string(".21.1.14.") + row_str ) );
 855                         if( o3.hasField("w") )
 856                             out_vals.insert( extract_uint64( o3["w"], string(".21.1.15.") + row_str ) );
 857                     }
 858
 859                     if( dbl.hasField("timeAcquiringMicros") )
 860                     {
 861                         BSONObj o3 = dbl["timeAcquiringMicros"].Obj();
 862
 863                         if( o3.hasField("r") )
 864                             out_vals.insert( extract_uint64( o3["r"], string(".21.1.16.") + row_str ) );
 865                         if( o3.hasField("w") )
 866                             out_vals.insert( extract_uint64( o3["w"], string(".21.1.17.") + row_str ) );
 867                     }
 868                 }
 869 */
 870     extractor_map->insert( make_pair<string, Extractor *>( "locks", locks_extractors(dbnames) ) );
 871     extractor_map->insert( make_pair<string, Extractor *>( "repl", serv_info_repl_extractors() ) );
 872     extractor_map->insert( make_pair<string, Extractor *>( "replNetworkQueue", repl_network_queue_extractors() ) );
 873     // extractor_map->insert( make_pair<string, Extractor *>( "indexCounters", index_cOunters_extractors() ) );
 874
 875     return new StructExtractor(extractor_map);
 876 }
 877
 878 template<class T1, class T2>
 879 struct DualItemExtractor
 880     : public Extractor
 881 {
 882 public:
 883     DualItemExtractor(string const &oid1, string const &oid2)
 884         : Extractor()
 885         , m_oid1(oid1)
 886         , m_oid2(oid2)
 887     {}
 888
 889     virtual void operator()(BSONElement const &e, set<OidValueTuple> &out_vals)
 890     {
 891         out_vals.insert( extract<T1>( e, m_oid1 ) );
 892         out_vals.insert( extract<T2>( e, m_oid2 ) );
 893     }
 894
 895 protected:
 896     string const m_oid1;
 897     string const m_oid2;
 898
 899 private:
 900     DualItemExtractor();
 901 };
 902
 903 struct ReplSetMemberRowExtractor
 904     : public StructExtractor
 905 {
 906 public:
 907     ReplSetMemberRowExtractor()
 908         : StructExtractor(get_extractor_map())
 909     {}
 910
 911 protected:
 912     static map<string, Extractor *> *
 913     get_extractor_map()
 914     {
 915         map<string, Extractor *> *extractor_map = new map<string, Extractor *>;
 916
 917         extractor_map->insert( make_pair<string, Extractor *>( "_id", new ItemExtractor<unsigned>( ".1" ) ) );
 918         extractor_map->insert( make_pair<string, Extractor *>( "name", new ItemExtractor<string>( ".2" ) ) );
 919         extractor_map->insert( make_pair<string, Extractor *>( "health", new ItemExtractor<double>( ".3" ) ) );
 920         extractor_map->insert( make_pair<string, Extractor *>( "state", new ItemExtractor<unsigned>( ".4" ) ) );
 921         extractor_map->insert( make_pair<string, Extractor *>( "stateStr", new ItemExtractor<string>( ".5" ) ) );
 922         extractor_map->insert( make_pair<string, Extractor *>( "uptime", new ItemExtractor<unsigned long long>( ".6" ) ) );
 923         extractor_map->insert( make_pair<string, Extractor *>(
 924             "optime", new DualItemExtractor<unsigned long long, unsigned>( ".7", ".8" ) ) );
 925         extractor_map->insert( make_pair<string, Extractor *>( "pingMs", new ItemExtractor<unsigned long long>( ".9" ) ) );
 926         extractor_map->insert( make_pair<string, Extractor *>( "lastHeartbeat", new ItemExtractor<unsigned long long>( ".10" ) ) );
 927
 928         return extractor_map;
 929     }
 930 };
 931
 932 StructExtractor *
 933 repl_set_status_extractors()
 934 {
 935     map<string, Extractor *> *extractor_map = new map<string, Extractor *>;
 936
 937     static RowPostfix repl_rows;
 938     vector<string> repl_tbl;
 939     repl_tbl.push_back(".2");
 940     extractor_map->insert( make_pair<string, Extractor *>(
 941         "members", new ListRowExtractor<ReplSetMemberRowExtractor>(".20.7", repl_rows, repl_tbl) ) );
 942
 943     return new StructExtractor(extractor_map);
 944 }
 945
 946 struct DatabasesMemberRowExtractor
 947     : public StructExtractor
 948 {
 949 public:
 950     DatabasesMemberRowExtractor()
 951         : StructExtractor(get_extractor_map())
 952         , m_conn(NULL)
 953         , m_cmd(BSONObjBuilder().append("dbstats", 1).obj())
 954         , m_dbinfo()
 955         , m_dbextractor(get_dbinfo_map())
 956     {}
 957
 958     DatabasesMemberRowExtractor(DBClientConnection *conn)
 959         : StructExtractor(get_extractor_map())
 960         , m_conn(conn)
 961         , m_cmd(BSONObjBuilder().append("dbstats", 1).obj())
 962         , m_dbinfo()
 963         , m_dbextractor(get_dbinfo_map())
 964     {}
 965
 966     virtual void operator()(BSONElement const &e, set<OidValueTuple> &out_vals)
 967     {
 968         StructExtractor::operator() (e, out_vals);
 969
 970         OidValueTuple search_key( ".1" ); // ASN.1 type doesn't matter, only OID
 971         set<OidValueTuple>::iterator cmp_iter = out_vals.lower_bound(search_key);
 972         if( m_conn && (cmp_iter != out_vals.end()) )
 973         {
 974             m_conn->runCommand(cmp_iter->value, m_cmd, m_dbinfo);
 975             m_dbextractor(m_dbinfo, out_vals);
 976         }
 977     }
 978
 979     virtual void operator()(BSONObj const &o, set<OidValueTuple> &out_vals)
 980     {
 981         StructExtractor::operator() (o, out_vals);
 982
 983         OidValueTuple search_key( ".1" ); // ASN.1 type doesn't matter, only OID
 984         set<OidValueTuple>::iterator cmp_iter = out_vals.lower_bound(search_key);
 985         if( m_conn && (cmp_iter != out_vals.end()) )
 986         {
 987             m_conn->runCommand(cmp_iter->value, m_cmd, m_dbinfo);
 988             m_dbextractor(m_dbinfo, out_vals);
 989         }
 990     }
 991
 992     void set_conn(DBClientConnection *conn) { m_conn = conn; }
 993
 994 protected:
 995     DBClientConnection *m_conn;
 996     BSONObj m_cmd, m_dbinfo;
 997     StructExtractor m_dbextractor;
 998
 999     static map<string, Extractor *> *
1000     get_extractor_map()
1001     {
1002         map<string, Extractor *> *extractor_map = new map<string, Extractor *>;
1003
1004         extractor_map->insert( make_pair<string, Extractor *>( "name", new ItemExtractor<string>( ".1" ) ) );
1005         extractor_map->insert( make_pair<string, Extractor *>( "sizeOnDisk", new ItemExtractor<unsigned long long>( ".2" ) ) );
1006         extractor_map->insert( make_pair<string, Extractor *>( "empty", new ItemExtractor<int>( ".3" ) ) );
1007
1008         return extractor_map;
1009     }
1010
1011     static map<string, Extractor *> *
1012     get_dbinfo_map()
1013     {
1014         map<string, Extractor *> *extractor_map = new map<string, Extractor *>;
1015
1016         extractor_map->insert( make_pair<string, Extractor *>( "collections", new ItemExtractor<unsigned long long>( ".4" ) ) );
1017         extractor_map->insert( make_pair<string, Extractor *>( "objects", new ItemExtractor<unsigned long long>( ".5" ) ) );
1018         extractor_map->insert( make_pair<string, Extractor *>( "avgObjSize", new ItemExtractor<double>( ".6" ) ) );
1019         extractor_map->insert( make_pair<string, Extractor *>( "dataSize", new ItemExtractor<unsigned long long>( ".7" ) ) );
1020         extractor_map->insert( make_pair<string, Extractor *>( "storageSize", new ItemExtractor<unsigned long long>( ".8" ) ) );
1021         extractor_map->insert( make_pair<string, Extractor *>( "numExtents", new ItemExtractor<unsigned>( ".9" ) ) );
1022         extractor_map->insert( make_pair<string, Extractor *>( "indexes", new ItemExtractor<unsigned>( ".10" ) ) );
1023         extractor_map->insert( make_pair<string, Extractor *>( "sizeOnDisk", new ItemExtractor<unsigned>( ".11" ) ) );
1024         extractor_map->insert( make_pair<string, Extractor *>( "fileSize", new ItemExtractor<unsigned long long>( ".12" ) ) );
1025         extractor_map->insert( make_pair<string, Extractor *>( "nsSizeMB", new ItemExtractor<unsigned>( ".13" ) ) );
1026
1027         return extractor_map;
1028     }
1029 };
1030
1031 StructExtractor *
1032 databases_extractors(DBClientConnection &conn)
1033 {
1034     map<string, Extractor *> *extractor_map = new map<string, Extractor *>;
1035
1036     static RowPostfix db_rows;
1037     vector<string> db_tbl;
1038     db_tbl.push_back(".1");
1039     ListRowExtractor<DatabasesMemberRowExtractor> *lre = new ListRowExtractor<DatabasesMemberRowExtractor>(".21", db_rows, db_tbl);
1040     lre->set_conn(&conn);
1041     extractor_map->insert( make_pair<string, Extractor *>( "databases", lre ) );
1042
1043     return new StructExtractor(extractor_map);
1044 }
1045
1046 void
1047 collect(DBClientConnection &c, set<OidValueTuple> &out_vals)
1048 {
1049     BSONObj serv_status, dbases, repl_info, cmd;
1050     StructExtractor *bson_extractor = 0;
1051
1052     cmd = BSONObjBuilder().append("listDatabases", 1).obj();
1053     c.runCommand(DBNAME, cmd, dbases);
1054
1055     bson_extractor = databases_extractors(c);
1056     (*bson_extractor)(dbases, out_vals);
1057
1058     // XXX extract row + dbname for serv_status.locks[]
1059     vector<string> database_names;
1060     OidValueTuple search_key(".21.1.1.");
1061     for( set<OidValueTuple>::iterator cmp_iter = out_vals.lower_bound(search_key);
1062          ( cmp_iter != out_vals.end() ) && ( 0 == cmp_iter->oid.compare( 0, search_key.oid.length(), search_key.oid ) );
1063          ++cmp_iter )
1064     {
1065         database_names.push_back(cmp_iter->value);
1066     }
1067
1068     cmd = BSONObjBuilder().append( "serverStatus", 1 ).obj();
1069     c.runCommand(DBNAME, cmd, serv_status);
1070
1071     bson_extractor = server_status_extractors(database_names);
1072     (*bson_extractor)(serv_status, out_vals);
1073
1074     cmd = BSONObjBuilder().append("replSetGetStatus", 1).obj();
1075     c.runCommand(DBNAME, cmd, repl_info);
1076
1077     bson_extractor = repl_set_status_extractors();
1078     (*bson_extractor)(repl_info, out_vals);
1079 }
1080
1081 void
1082 dump(set<OidValueTuple> const &out_vals)
1083 {
1084     vector<string> result;
1085     result.reserve(out_vals.size() + 2);
1086     for( set<OidValueTuple>::iterator iter = out_vals.begin();
1087          iter != out_vals.end();
1088          ++iter )
1089     {
1090         std::string s = "  [ \"";
1091
1092         s += iter->oid;
1093         s += "\", ";
1094         s += lexical_cast<string>(iter->type);
1095         s += ", ";
1096         if( ASN_OCTET_STR == iter->type )
1097             s += "\"";
1098         if( ASN_NULL == iter->type )
1099             s += "null";
1100         else if( ASN_OCTET_STR == iter->type )
1101             s += boost::locale::conv::utf_to_utf<char>(iter->value);
1102         else
1103             s += iter->value;
1104         if( ASN_OCTET_STR == iter->type )
1105             s += "\"";
1106         s += " ]";
1107
1108         result.push_back(s);
1109     }
1110
1111     cout << "[" << endl
1112          << join(",\n", result) << endl
1113          << "]" << endl;
1114 }
1115
1116 int
1117 main(int argc, char *argv[])
1118 {
1119     try
1120     {
1121         options_description desc("Allowed options");
1122         desc.add_options()
1123             ("help", "produce help message")
1124             ("dsn", value<string>(), "set mongodb dsn")
1125             ;
1126         variables_map vm;
1127         store( parse_command_line( argc, argv, desc ), vm );
1128         notify(vm);
1129
1130         if( vm.count("help") )
1131         {
1132             cout << desc << endl;
1133             return 1;
1134         }
1135
1136         if( vm.count("dsn") == 0 )
1137         {
1138             cerr << desc << endl;
1139             return 255;
1140         }
1141
1142         set<OidValueTuple> out_vals;
1143         do {
1144             DBClientConnection c;
1145
1146             boost::timer::cpu_timer db_dur;
1147             db_dur.start();
1148             connect(c, vm["dsn"].as<string>(), DBNAME, "admin");
1149             collect(c, out_vals);
1150             db_dur.stop();
1151
1152             OidValueTuple val( ".99.1", SMI_COUNTER64 );
1153             val.value = lexical_cast<string>( db_dur.elapsed().user );
1154             out_vals.insert( val );
1155
1156             val.oid = ".99.2";
1157             val.value = lexical_cast<string>( db_dur.elapsed().system );
1158             out_vals.insert( val );
1159
1160             val.oid = ".99.3";
1161             val.value = lexical_cast<string>( db_dur.elapsed().wall );
1162             out_vals.insert( val );
1163         } while(0);
1164
1165         dump(out_vals);
1166     }
1167     catch( DBException &e )
1168     {
1169         cout << "caught " << e.what() << endl;
1170     }
1171
1172     return 0;
1173 }