src/CurlWikiGrabber.cpp

   1 #include "CurlWikiGrabber.h"
   2
   3 #ifndef DEBUG
   4 #define NDEBUG
   5 #endif
   6 #include <cassert>
   7
   8 static size_t write_callback(char *ptr, size_t size, size_t nmemb, void *userdata)
   9 {
  10     static_cast<std::string*>(userdata)->append(ptr, size * nmemb);
  11     return size * nmemb;
  12 }
  13
  14 CurlWikiGrabber::CurlWikiGrabber()
  15 {
  16     int error = curl_global_init(CURL_GLOBAL_ALL);
  17     if(error) {
  18         throw WalkerException("CURL init failed");
  19     }
  20 }
  21
  22 //! \todo Curl return code checking
  23 std::string CurlWikiGrabber::grabUrl(std::string url) const
  24 {
  25     CURL *handle = curl_easy_init();
  26
  27     if(NULL == handle) {
  28         throw WalkerException("error initiating curl");
  29     }
  30
  31     CURLcode crv = curl_easy_setopt(handle, CURLOPT_URL, url.c_str());
  32     assert(crv == 0);
  33     crv = curl_easy_setopt(handle, CURLOPT_USERAGENT, "WikiWalker/ test program");
  34     assert(crv == 0);
  35     crv = curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, write_callback);
  36     assert(crv == 0);
  37
  38     std::string gotContent;
  39     crv = curl_easy_setopt(handle, CURLOPT_WRITEDATA, &gotContent);
  40     assert(crv == 0);
  41
  42     gotContent = "";
  43     crv = curl_easy_perform(handle);
  44
  45     if(crv != 0) {
  46         const char* err = curl_easy_strerror(crv);
  47         std::string text(err);
  48         throw WalkerException(text);
  49     }
  50
  51     long httpcode = 0;
  52     crv = curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &httpcode);
  53     assert(crv == 0);
  54
  55     curl_easy_cleanup(handle);
  56
  57     handle = NULL;
  58
  59     if(httpcode != 200) {
  60         return "";
  61     }
  62
  63     return gotContent;
  64 }
  65
  66 // note to self: API
  67 // https://en.wikipedia.org/w/api.php
  68 // /w/api.php?action=query&format=json&prop=links&plnamespace=0&titles=<title>
  69 // maybe &pllimit=100