src/CurlWikiGrabber.cpp

   1 //! \file CurlWikiGrabber.cpp
   2
   3 #include "CurlWikiGrabber.h"
   4 #include "WalkerException.h"
   5
   6 #ifndef DEBUG
   7 #define NDEBUG
   8 #endif
   9 #include <cassert>
  10
  11 static size_t write_callback(char *ptr, size_t size, size_t nmemb, void *userdata)
  12 {
  13     static_cast<std::string*>(userdata)->append(ptr, size * nmemb);
  14     return size * nmemb;
  15 }
  16
  17 CurlWikiGrabber::CurlWikiGrabber()
  18 {
  19     int error = curl_global_init(CURL_GLOBAL_ALL);
  20     if(error) {
  21         throw WalkerException("CURL init failed");
  22     }
  23 }
  24
  25 //! \todo Curl return code checking
  26 std::string CurlWikiGrabber::grabUrl(std::string url) const
  27 {
  28     CURL *handle = curl_easy_init();
  29
  30     if(NULL == handle) {
  31         throw WalkerException("error initiating curl");
  32     }
  33
  34     CURLcode crv = curl_easy_setopt(handle, CURLOPT_URL, url.c_str());
  35     assert(crv == 0);
  36     crv = curl_easy_setopt(handle, CURLOPT_USERAGENT, "WikiWalker/ test program");
  37     assert(crv == 0);
  38     crv = curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, write_callback);
  39     assert(crv == 0);
  40
  41     std::string gotContent;
  42     crv = curl_easy_setopt(handle, CURLOPT_WRITEDATA, &gotContent);
  43     assert(crv == 0);
  44
  45     gotContent = "";
  46     crv = curl_easy_perform(handle);
  47
  48     if(crv != 0) {
  49         const char* err = curl_easy_strerror(crv);
  50         std::string text(err);
  51         throw WalkerException(text);
  52     }
  53
  54     long httpcode = 0;
  55     crv = curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &httpcode);
  56     assert(crv == 0);
  57
  58     curl_easy_cleanup(handle);
  59
  60     handle = NULL;
  61
  62     if(httpcode != 200) {
  63         return "";
  64     }
  65
  66     return gotContent;
  67 }
  68
  69 // note to self: API
  70 // https://en.wikipedia.org/w/api.php
  71 // /w/api.php?action=query&format=json&prop=links&plnamespace=0&titles=<title>
  72 // maybe &pllimit=100