components/dom_distiller/core/distiller.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/dom_distiller/core/distiller.h"
   6
   7 #include <map>
   8 #include <vector>
   9
  10 #include "base/auto_reset.h"
  11 #include "base/bind.h"
  12 #include "base/callback.h"
  13 #include "base/location.h"
  14 #include "base/message_loop/message_loop.h"
  15 #include "base/strings/string_number_conversions.h"
  16 #include "base/strings/utf_string_conversions.h"
  17 #include "base/values.h"
  18 #include "components/dom_distiller/core/distiller_page.h"
  19 #include "components/dom_distiller/core/distiller_url_fetcher.h"
  20 #include "components/dom_distiller/core/proto/distilled_article.pb.h"
  21 #include "components/dom_distiller/core/proto/distilled_page.pb.h"
  22 #include "net/url_request/url_request_context_getter.h"
  23
  24 namespace {
  25 // Maximum number of distilled pages in an article.
  26 const size_t kMaxPagesInArticle = 32;
  27 }
  28
  29 namespace dom_distiller {
  30
  31 DistillerFactoryImpl::DistillerFactoryImpl(
  32     scoped_ptr<DistillerURLFetcherFactory> distiller_url_fetcher_factory,
  33     const dom_distiller::proto::DomDistillerOptions& dom_distiller_options)
  34     : distiller_url_fetcher_factory_(distiller_url_fetcher_factory.Pass()),
  35       dom_distiller_options_(dom_distiller_options) {
  36 }
  37
  38 DistillerFactoryImpl::~DistillerFactoryImpl() {}
  39
  40 scoped_ptr<Distiller> DistillerFactoryImpl::CreateDistiller() {
  41   scoped_ptr<DistillerImpl> distiller(new DistillerImpl(
  42       *distiller_url_fetcher_factory_, dom_distiller_options_));
  43   return distiller.PassAs<Distiller>();
  44 }
  45
  46 DistillerImpl::DistilledPageData::DistilledPageData() {}
  47
  48 DistillerImpl::DistilledPageData::~DistilledPageData() {}
  49
  50 DistillerImpl::DistillerImpl(
  51     const DistillerURLFetcherFactory& distiller_url_fetcher_factory,
  52     const dom_distiller::proto::DomDistillerOptions& dom_distiller_options)
  53     : distiller_url_fetcher_factory_(distiller_url_fetcher_factory),
  54       dom_distiller_options_(dom_distiller_options),
  55       max_pages_in_article_(kMaxPagesInArticle),
  56       destruction_allowed_(true),
  57       weak_factory_(this) {
  58 }
  59
  60 DistillerImpl::~DistillerImpl() {
  61   DCHECK(destruction_allowed_);
  62 }
  63
  64 void DistillerImpl::SetMaxNumPagesInArticle(size_t max_num_pages) {
  65   max_pages_in_article_ = max_num_pages;
  66 }
  67
  68 bool DistillerImpl::AreAllPagesFinished() const {
  69   return started_pages_index_.empty() && waiting_pages_.empty();
  70 }
  71
  72 size_t DistillerImpl::TotalPageCount() const {
  73   return waiting_pages_.size() + started_pages_index_.size() +
  74          finished_pages_index_.size();
  75 }
  76
  77 void DistillerImpl::AddToDistillationQueue(int page_num, const GURL& url) {
  78   if (!IsPageNumberInUse(page_num) && url.is_valid() &&
  79       TotalPageCount() < max_pages_in_article_ &&
  80       seen_urls_.find(url.spec()) == seen_urls_.end()) {
  81     waiting_pages_[page_num] = url;
  82   }
  83 }
  84
  85 bool DistillerImpl::IsPageNumberInUse(int page_num) const {
  86   return waiting_pages_.find(page_num) != waiting_pages_.end() ||
  87          started_pages_index_.find(page_num) != started_pages_index_.end() ||
  88          finished_pages_index_.find(page_num) != finished_pages_index_.end();
  89 }
  90
  91 DistillerImpl::DistilledPageData* DistillerImpl::GetPageAtIndex(size_t index)
  92     const {
  93   DCHECK_LT(index, pages_.size());
  94   DistilledPageData* page_data = pages_[index];
  95   DCHECK(page_data);
  96   return page_data;
  97 }
  98
  99 void DistillerImpl::DistillPage(const GURL& url,
 100                                 scoped_ptr<DistillerPage> distiller_page,
 101                                 const DistillationFinishedCallback& finished_cb,
 102                                 const DistillationUpdateCallback& update_cb) {
 103   DCHECK(AreAllPagesFinished());
 104   distiller_page_ = distiller_page.Pass();
 105   finished_cb_ = finished_cb;
 106   update_cb_ = update_cb;
 107
 108   AddToDistillationQueue(0, url);
 109   DistillNextPage();
 110 }
 111
 112 void DistillerImpl::DistillNextPage() {
 113   if (!waiting_pages_.empty()) {
 114     std::map<int, GURL>::iterator front = waiting_pages_.begin();
 115     int page_num = front->first;
 116     const GURL url = front->second;
 117
 118     waiting_pages_.erase(front);
 119     DCHECK(url.is_valid());
 120     DCHECK(started_pages_index_.find(page_num) == started_pages_index_.end());
 121     DCHECK(finished_pages_index_.find(page_num) == finished_pages_index_.end());
 122     seen_urls_.insert(url.spec());
 123     pages_.push_back(new DistilledPageData());
 124     started_pages_index_[page_num] = pages_.size() - 1;
 125     distiller_page_->DistillPage(
 126         url,
 127         dom_distiller_options_,
 128         base::Bind(&DistillerImpl::OnPageDistillationFinished,
 129                    weak_factory_.GetWeakPtr(),
 130                    page_num,
 131                    url));
 132   }
 133 }
 134
 135 void DistillerImpl::OnPageDistillationFinished(
 136     int page_num,
 137     const GURL& page_url,
 138     scoped_ptr<proto::DomDistillerResult> distiller_result,
 139     bool distillation_successful) {
 140   DCHECK(started_pages_index_.find(page_num) != started_pages_index_.end());
 141   if (distillation_successful) {
 142     DCHECK(distiller_result.get());
 143     DistilledPageData* page_data =
 144         GetPageAtIndex(started_pages_index_[page_num]);
 145     page_data->distilled_page_proto =
 146         new base::RefCountedData<DistilledPageProto>();
 147     page_data->page_num = page_num;
 148     if (distiller_result->has_title()) {
 149       page_data->distilled_page_proto->data.set_title(
 150           distiller_result->title());
 151     }
 152     page_data->distilled_page_proto->data.set_url(page_url.spec());
 153     if (distiller_result->has_distilled_content() &&
 154         distiller_result->distilled_content().has_html()) {
 155       page_data->distilled_page_proto->data.set_html(
 156           distiller_result->distilled_content().html());
 157     }
 158
 159     if (distiller_result->has_pagination_info()) {
 160       proto::PaginationInfo pagination_info =
 161           distiller_result->pagination_info();
 162       if (pagination_info.has_next_page()) {
 163         GURL next_page_url(pagination_info.next_page());
 164         if (next_page_url.is_valid()) {
 165           // The pages should be in same origin.
 166           DCHECK_EQ(next_page_url.GetOrigin(), page_url.GetOrigin());
 167           AddToDistillationQueue(page_num + 1, next_page_url);
 168         }
 169       }
 170
 171       if (pagination_info.has_prev_page()) {
 172         GURL prev_page_url(pagination_info.prev_page());
 173         if (prev_page_url.is_valid()) {
 174           DCHECK_EQ(prev_page_url.GetOrigin(), page_url.GetOrigin());
 175           AddToDistillationQueue(page_num - 1, prev_page_url);
 176         }
 177       }
 178     }
 179
 180     for (int img_num = 0; img_num < distiller_result->image_urls_size();
 181          ++img_num) {
 182       std::string image_id =
 183           base::IntToString(page_num + 1) + "_" + base::IntToString(img_num);
 184       FetchImage(page_num, image_id, distiller_result->image_urls(img_num));
 185     }
 186
 187     AddPageIfDone(page_num);
 188     DistillNextPage();
 189   } else {
 190     started_pages_index_.erase(page_num);
 191     RunDistillerCallbackIfDone();
 192   }
 193 }
 194
 195 void DistillerImpl::FetchImage(int page_num,
 196                                const std::string& image_id,
 197                                const std::string& item) {
 198   DCHECK(started_pages_index_.find(page_num) != started_pages_index_.end());
 199   DistilledPageData* page_data = GetPageAtIndex(started_pages_index_[page_num]);
 200   DistillerURLFetcher* fetcher =
 201       distiller_url_fetcher_factory_.CreateDistillerURLFetcher();
 202   page_data->image_fetchers_.push_back(fetcher);
 203
 204   fetcher->FetchURL(item,
 205                     base::Bind(&DistillerImpl::OnFetchImageDone,
 206                                weak_factory_.GetWeakPtr(),
 207                                page_num,
 208                                base::Unretained(fetcher),
 209                                image_id));
 210 }
 211
 212 void DistillerImpl::OnFetchImageDone(int page_num,
 213                                      DistillerURLFetcher* url_fetcher,
 214                                      const std::string& id,
 215                                      const std::string& response) {
 216   DCHECK(started_pages_index_.find(page_num) != started_pages_index_.end());
 217   DistilledPageData* page_data = GetPageAtIndex(started_pages_index_[page_num]);
 218   DCHECK(page_data->distilled_page_proto.get());
 219   DCHECK(url_fetcher);
 220   ScopedVector<DistillerURLFetcher>::iterator fetcher_it =
 221       std::find(page_data->image_fetchers_.begin(),
 222                 page_data->image_fetchers_.end(),
 223                 url_fetcher);
 224
 225   DCHECK(fetcher_it != page_data->image_fetchers_.end());
 226   // Delete the |url_fetcher| by DeleteSoon since the OnFetchImageDone
 227   // callback is invoked by the |url_fetcher|.
 228   page_data->image_fetchers_.weak_erase(fetcher_it);
 229   base::MessageLoop::current()->DeleteSoon(FROM_HERE, url_fetcher);
 230
 231   DistilledPageProto_Image* image =
 232       page_data->distilled_page_proto->data.add_image();
 233   image->set_name(id);
 234   image->set_data(response);
 235
 236   AddPageIfDone(page_num);
 237 }
 238
 239 void DistillerImpl::AddPageIfDone(int page_num) {
 240   DCHECK(started_pages_index_.find(page_num) != started_pages_index_.end());
 241   DCHECK(finished_pages_index_.find(page_num) == finished_pages_index_.end());
 242   DistilledPageData* page_data = GetPageAtIndex(started_pages_index_[page_num]);
 243   if (page_data->image_fetchers_.empty()) {
 244     finished_pages_index_[page_num] = started_pages_index_[page_num];
 245     started_pages_index_.erase(page_num);
 246     const ArticleDistillationUpdate& article_update =
 247         CreateDistillationUpdate();
 248     DCHECK_EQ(article_update.GetPagesSize(), finished_pages_index_.size());
 249     update_cb_.Run(article_update);
 250     RunDistillerCallbackIfDone();
 251   }
 252 }
 253
 254 const ArticleDistillationUpdate DistillerImpl::CreateDistillationUpdate()
 255     const {
 256   bool has_prev_page = false;
 257   bool has_next_page = false;
 258   if (!finished_pages_index_.empty()) {
 259     int prev_page_num = finished_pages_index_.begin()->first - 1;
 260     int next_page_num = finished_pages_index_.rbegin()->first + 1;
 261     has_prev_page = IsPageNumberInUse(prev_page_num);
 262     has_next_page = IsPageNumberInUse(next_page_num);
 263   }
 264
 265   std::vector<scoped_refptr<ArticleDistillationUpdate::RefCountedPageProto> >
 266       update_pages;
 267   for (std::map<int, size_t>::const_iterator it = finished_pages_index_.begin();
 268        it != finished_pages_index_.end();
 269        ++it) {
 270     update_pages.push_back(pages_[it->second]->distilled_page_proto);
 271   }
 272   return ArticleDistillationUpdate(update_pages, has_next_page, has_prev_page);
 273 }
 274
 275 void DistillerImpl::RunDistillerCallbackIfDone() {
 276   DCHECK(!finished_cb_.is_null());
 277   if (AreAllPagesFinished()) {
 278     bool first_page = true;
 279     scoped_ptr<DistilledArticleProto> article_proto(
 280         new DistilledArticleProto());
 281     // Stitch the pages back into the article.
 282     for (std::map<int, size_t>::iterator it = finished_pages_index_.begin();
 283          it != finished_pages_index_.end();) {
 284       DistilledPageData* page_data = GetPageAtIndex(it->second);
 285       *(article_proto->add_pages()) = page_data->distilled_page_proto->data;
 286
 287       if (first_page) {
 288         article_proto->set_title(page_data->distilled_page_proto->data.title());
 289         first_page = false;
 290       }
 291
 292       finished_pages_index_.erase(it++);
 293     }
 294
 295     pages_.clear();
 296     DCHECK_LE(static_cast<size_t>(article_proto->pages_size()),
 297               max_pages_in_article_);
 298
 299     DCHECK(pages_.empty());
 300     DCHECK(finished_pages_index_.empty());
 301
 302     base::AutoReset<bool> dont_delete_this_in_callback(&destruction_allowed_,
 303                                                        false);
 304     finished_cb_.Run(article_proto.Pass());
 305     finished_cb_.Reset();
 306   }
 307 }
 308
 309 }  // namespace dom_distiller