Remove old about scheme URL constants.

[chromium-blink-merge.git] / chrome / tools / webforms_aggregator.py

#!/usr/bin/env python
# Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Downloads web pages with fillable forms after parsing through a set of links.

Used for collecting web pages with forms. Used as a standalone script.
This script assumes that it's run from within the same directory in which it's
checked into. If this script were to be run elsewhere then the path for
REGISTER_PAGE_DIR needs to be changed.

This script assumes that third party modules are installed:
httplib2, lxml, pycurl.

Usage: webforms_aggregator.py [options] [single url or file containing urls]

Options:
  -l LOG_LEVEL, --log_level LOG_LEVEL
    LOG_LEVEL: debug, info, warning or error [default: error]
  -h, --help  show this help message and exit
"""

import datetime
import errno
import logging
import optparse
import os
import re
# Needed in Linux so that PyCurl does not throw a segmentation fault.
import signal
import sys
import tempfile
import threading
import time
import urlparse

import httplib2
from lxml import html, etree
import pycurl

REGISTER_PAGE_DIR = os.path.join(os.pardir, 'test', 'data', 'autofill',
                                 'heuristics', 'input')
NOT_FOUND_REG_PAGE_SITES_FILENAME = 'notFoundRegPageSites.txt'

FORM_LOCATION_COMMENT = 'Form Location: %s'
HTML_FILE_PREFIX = 'grabber-'

MAX_REDIRECTIONS = 10

# Strings in a webpage that are indicative of a registration link.
LINK_CLUES = ['regist', 'user', 'sign', 'login', 'account']

MAX_SAME_DOMAIN_URLS_NO = 30
MAX_TOTAL_URLS_PER_DOMAIN = 300
MAX_OPEN_FILES_NO = 500

# URLs are selected for downloading with the following rules from the link
# lists, giving more weight to the links that contain a link clue.
CLUE_SECURE_LINKS_NO = MAX_SAME_DOMAIN_URLS_NO * 3/10
CLUE_GENERAL_LINKS_NO = MAX_SAME_DOMAIN_URLS_NO * 3/10
SECURE_LINKS_NO = MAX_SAME_DOMAIN_URLS_NO * 2/10
GENERAL_LINKS_NO = MAX_SAME_DOMAIN_URLS_NO * 2/10

MAX_ALLOWED_THREADS = MAX_OPEN_FILES_NO / MAX_SAME_DOMAIN_URLS_NO + 1


class Retriever(object):
  """Download, parse, and check if the web page contains a registration form.

  The objects of this class has a one to one relation with the web pages. For
  each page that is downloaded and parsed an object of this class is created.
  Each Retriever object creates a curl object. This object is added to the curl
  multi object of the crawler object so that the corresponding pages gets
  downloaded.
  """
  logger = logging.getLogger(__name__)

  def __init__(self, url, domain, cookie_file):
    """Initializes a Retriever object.

    Args:
      url: url to download page from.
      domain: only links with this domain will be retrieved.
      cookie_file: the name of a cookie file, needed for pages that use session
          cookies to change their contents.
    """
    self._url = url
    self._domain = domain
    self._html_content = ''

    # Http links without clues from LINK_CLUES.
    self._general_links = []
    # Http links that contain a clue from LINK_CLUES.
    self._clues_general_links = []
    # Https links that do not contain any clues from LINK_CLUES.
    self._secure_links = []
    # Https links that contain a clue from LINK_CLUES.
    self._clues_secure_links = []
    self._cookie_file = cookie_file
    self._curl_object = None

  def __del__(self):
    """Cleans up before this object is destroyed.

    The function closes the corresponding curl object that does the downloading.
    """
    if self._curl_object:
      self._curl_object.close()

  def _AddLink(self, link):
    """Adds url |link|, if not already present, to the appropriate list.

    The link only gets added to the single list that is appopriate for it:
    _secure_links, _general_links, _clues_secure_links or _clues_general_links.

    Args:
      link: the url that is inserted to the appropriate links list.
    """
    # Handles sites with unicode URLs.
    if isinstance(link, unicode):
      # Encode in 'utf-8' to avoid the UnicodeEncodeError exception.
      link = httplib2.iri2uri(link).encode('utf-8')
    link_parsed = urlparse.urlparse(link)
    link_lists = [self._clues_secure_links, self._secure_links,
                  self._clues_general_links, self._general_links]
    # Checks that the registration page is within the domain.
    if (self._domain in link_parsed[1] and
        all(link not in x for x in link_lists)):
      for clue in LINK_CLUES:
        if clue in link.lower():
          if link_parsed[0].startswith('https'):
            self._clues_secure_links.append(link)
            return
          else:
            self._clues_general_links.append(link)
            return
      if link_parsed[0].startswith('https'):  # No clues found in the link.
        self._secure_links.append(link)
      else:
        self._general_links.append(link)

  def ParseAndGetLinks(self):
    """Parses downloaded page and gets url link for non registration page.

    Checks if current page contains a registration page and if not it gets
    the url links. If it is a registration page, it saves it in a file as
    'grabber-' + domain + '.html' after it has added the FORM_LOCATION_COMMENT
    and it returns True. Otherwise it returns False.

    Returns:
      True if current page contains a registration form, and False otherwise.

    Raises:
      IOError: When can't write to the file.
    """
    if not self._domain:
      self.logger.error('Error: self._domain was not set')
      sys.exit(1)
    match_list = re.findall(r'(?P<quote>[\'\"])(?P<link>(?:https?:)?//.*?)\1',
                             self._html_content)
    for group_list in match_list:
      link = group_list[1]
      if link.startswith('//'):
        link = urlparse.urljoin(self._url, link)
      self._AddLink(link)
    try:
      tree = html.fromstring(self._html_content, parser=html.HTMLParser())
    except etree.LxmlError:
      self.logger.info('\t\tSkipping: not valid HTML code in this page <<< %s',
                       self._url)
      return False
    try:
      body = tree.iter('body').next()
    except StopIteration:
      self.logger.info('\t\tSkipping: no "BODY" tag in this page <<< %s',
                       self._url)
      return False

    # Get a list of all input elements with attribute type='password'
    password_elements = list(body.iterfind('.//input[@type="password"]'))
    # Check for multiple password elements to distinguish between a login form
    # and a registration form (Password field and Confirm Password field).
    if password_elements and len(password_elements) >= 2:
      form_elements = []
      for password_elem in password_elements:
        form_elem = password_elem.xpath('ancestor::form[1]')
        if not form_elem:
          continue
        if not form_elem[0] in form_elements:
          form_elements.append(form_elem[0])
        else:
          # Confirms that the page contains a registration form if two passwords
          # are contained in the same form for form_elem[0].
          if not os.path.isdir(REGISTER_PAGE_DIR):
            os.makedirs(REGISTER_PAGE_DIR)
          # Locate the HTML tag and insert the form location comment after it.
          html_tag = tree.iter('html').next()
          comment = etree.Comment(FORM_LOCATION_COMMENT % self._url)
          html_tag.insert(0, comment)
          # Create a new file and save the HTML registration page code.
          f = open('%s/%s%s.html' % (REGISTER_PAGE_DIR, HTML_FILE_PREFIX,
                                     self._domain), 'w')
          try:
            f.write(html.tostring(tree, pretty_print=True))
          except IOError as e:
            self.logger.error('Error: %s', e)
            raise
          finally:
            f.close()
          return True  # Registration page found.
    # Indicates page is not a registration page and links must be parsed.
    link_elements = list(body.iter('a'))
    for link_elem in link_elements:
      link = link_elem.get('href')
      if not link or '#' == link[0]:
        continue
      link = urlparse.urljoin(self._url, link)
      link_parsed = urlparse.urlparse(link)
      if not link_parsed[0].startswith('http'):
        continue
      self._AddLink(link)
    return False  # Registration page not found.

  def InitRequestHead(self):
    """Initializes curl object for a HEAD request.

    A HEAD request is initiated so that we can check from the headers if this is
    a valid HTML file. If it is not a valid HTML file, then we do not initiate a
    GET request, saving any unnecessary downloadings.
    """
    self._curl_object = pycurl.Curl()
    self._curl_object.setopt(pycurl.URL, self._url)
    # The following line fixes the GnuTLS package error that pycurl depends
    # on for getting https pages.
    self._curl_object.setopt(pycurl.SSLVERSION, pycurl.SSLVERSION_SSLv3)
    self._curl_object.setopt(pycurl.FOLLOWLOCATION, True)
    self._curl_object.setopt(pycurl.NOBODY, True)
    self._curl_object.setopt(pycurl.SSL_VERIFYPEER, False);
    self._curl_object.setopt(pycurl.MAXREDIRS, MAX_REDIRECTIONS)
    self._curl_object.setopt(pycurl.FAILONERROR, False)
    self._curl_object.setopt(pycurl.COOKIEFILE, self._cookie_file)
    self._curl_object.setopt(pycurl.COOKIEJAR, self._cookie_file)
    self._curl_object.setopt(pycurl.CONNECTTIMEOUT, 30)
    self._curl_object.setopt(pycurl.TIMEOUT, 300)
    self._curl_object.setopt(pycurl.NOSIGNAL, 1)

  def InitRequestGet(self):
    """Initializes curl object for a GET request.

    This is called only for valid HTML files. The Pycurl makes a GET request.
    The page begins to download, but since not all the data of the pages comes
    at once. When some of the data on the page is downloaded Pycurl will put
    this data in the buffer. The data is appended to the end of the page until
    everything is downloaded.
    """
    self._curl_object.setopt(pycurl.NOBODY, False)
    self._curl_object.setopt(
        pycurl.WRITEFUNCTION, lambda buff: setattr(
            self, '_html_content', self._html_content + buff))

  def Download(self):
    """Downloads the self._url page.

    It first does a HEAD request and then it proceeds to a GET request.
    It uses a curl object for a single download. This function is called only
    once for the initial url of a site when we still don't have more urls from a
    domain.

    Returns:
      True, if the downloaded page is valid HTML code, or False otherwise.
    """
    self.InitRequestHead()
    try:
      self._curl_object.perform()
    except pycurl.error as e:
      self.logger.error('Error: %s, url: %s', e, self._url)
      return False
    self._url = urlparse.urljoin(
        self._url, self._curl_object.getinfo(pycurl.EFFECTIVE_URL))
    content_type = self._curl_object.getinfo(pycurl.CONTENT_TYPE)
    if content_type and ('text/html' in content_type.lower()):
      self.InitRequestGet()
      try:
        self._curl_object.perform()
      except pycurl.error as e:
        self.logger.error('Error: %s, url: %s', e, self._url)
        return False
      return True
    else:
      self.logger.info('\tSkipping: Not an HTML page <<< %s', self._url)
      return False

  def Run(self):
    """Called only once for the initial url when we do not have more urls.

    Downloads the originally-specified site url, parses it and gets the links.

    Returns:
      True, if a registration page is found, and False otherwise.
    """
    if self.Download():
      if not self._domain:
        url_parsed = urlparse.urlparse(self._url)
        self._domain = url_parsed[1]
        if self._domain.startswith('www'):
          self._domain = '.'.join(self._domain.split('.')[1:])
      if self.ParseAndGetLinks():
        return True
    return False


class Crawler(object):
  """Crawls a site until a registration page is found or max level is reached.

  Creates, uses and destroys Retriever objects. Creates a cookie temp file
  needed for session cookies. It keeps track of 'visited links' and
  'links to visit' of the site. To do this it uses the links discovered from
  each Retriever object. Use Run() to crawl the site.
  """
  try:
    signal.signal(signal.SIGPIPE, signal.SIG_IGN)
  except ImportError:
    pass
  logger = logging.getLogger(__name__)

  def __init__(self, url, logging_level=None):
    """Init crawler URL, links lists, logger, and creates a cookie temp file.

    The cookie temp file is needed for session cookies.

    Args:
      url: the initial "seed" url of the site.
      logging_level: the desired verbosity level, default is None.
    """
    if logging_level:
      self.logger.setLevel(logging_level)

    self.url_error = False
    url_parsed = urlparse.urlparse(url)
    if not url_parsed[0].startswith('http'):
      self.logger.error(
          'Error: "%s" does not begin with http:// or https://', url)
      self.url_error = True
      return
    # Example: if url is 'http://www.example.com?name=john' then value [1] or
    # network location is 'www.example.com'.
    if not url_parsed[1]:
      self.logger.error('Error: "%s" is not a valid url', url)
      self.url_error = True
      return
    self._url = url
    self._domain = ''
    # Http links that contain a clue from LINK_CLUES.
    self._clues_general_links = []
    # Http links that do not contain any clue from LINK_CLUES.
    self._general_links = []
    # Https links that contain a clue from LINK_CLUES.
    self._clues_secure_links = []
    # Https links that do not contain any clue from LINK_CLUES.
    self._secure_links = []
    # All links downloaded and parsed so far.
    self._links_visited = []
    self._retrievers_list = []
    self._cookie_file = tempfile.NamedTemporaryFile(
        suffix='.cookie', delete=False)
    self._cookie_file.close()
    self._cookie_file = self._cookie_file.name  # Keep only the filename.

  def __del__(self):
    """Deletes cookie file when Crawler instances are destroyed."""
    if hasattr(self, '_cookie_file'):
      self.logger.info('Deleting cookie file %s ...', self._cookie_file)
      os.unlink(self._cookie_file)

  def _MultiPerform(self, curl_multi_object):
    """Performs concurrent downloads using a CurlMulti object.

    Args:
      curl_multi_object: a curl object that downloads multiple pages
          concurrently. The class of this object is |pycurl.CurlMulti|.
    """
    # Following code uses the example from section for the CurlMulti object
    # at http://pycurl.sourceforge.net/doc/curlmultiobject.html.
    while True:
      ret, no_handles = curl_multi_object.perform()
      if ret != pycurl.E_CALL_MULTI_PERFORM:
        break
    while no_handles:
      curl_multi_object.select(1.0)
      while True:
        ret, no_handles = curl_multi_object.perform()
        if ret != pycurl.E_CALL_MULTI_PERFORM:
          break

  def _GetLinksPages(self, curl_multi_object):
    """Downloads many pages concurrently using a CurlMulti Object.

    Creates many Retriever objects and adds them to a list. The constant
    MAX_SAME_DOMAIN_URLS_NO defines the number of pages that can be downloaded
    concurrently from the same domain using the pycurl multi object. It's
    currently set to 30 URLs. These URLs are taken from the links lists, which
    are from csl, gcl, sl, and gl. The rules define how many URLs are taken from
    each list during each iteration.

    Example of the rules:
      3/10 from csl results in 9 URLs
      3/10 from cgl results in 9 URLs
      2/10 from sl results in 6 URLs
      2/10 from gl results in 6 URLs

    Adding up the above URLs gives 30 URLs that can be downloaded concurrently.
    If these lists have fewer items than the defined rules, such as if a site
    does not contain any secure links, then csl and sl lists will be of 0 length
    and only 15 pages would be downloaded concurrently from the same domain.

    Since 30 URLs can be handled concurrently, the number of links taken from
    other lists can be increased. This means that we can take 24 links from the
    cgl list so that 24 from gfl + 6 from gl = 30 URLs. If the cgl list has less
    than 24 links, e.g. there are only 21 links, then only 9 links may be taken
    from gl so ) + 21 + 0 + 9 = 30.

    Args:
      curl_multi_object: Each Retriever object has a curl object which is
          added to the CurlMulti Object.
    """
    self._retrievers_list = []

    csl_no = min(CLUE_SECURE_LINKS_NO, len(self._clues_secure_links))
    cgl_no = min(CLUE_GENERAL_LINKS_NO, len(self._clues_general_links))
    sl_no = min(SECURE_LINKS_NO, len(self._secure_links))
    gl_no = min(GENERAL_LINKS_NO, len(self._general_links))

    # If some links within the list have fewer items than needed, the missing
    # links will be taken by the following priority: csl, cgl, sl, gl.
    # c: clues, s: secure, g: general, l: list.
    spare_links = MAX_SAME_DOMAIN_URLS_NO - (csl_no + sl_no + cgl_no + gl_no)
    if spare_links > 0:
      csl_no = min(csl_no + spare_links, len(self._clues_secure_links))
      spare_links = MAX_SAME_DOMAIN_URLS_NO - (csl_no + sl_no + cgl_no + gl_no)
    if spare_links > 0:
      cgl_no = min(cgl_no + spare_links, len(self._clues_general_links))
      spare_links = MAX_SAME_DOMAIN_URLS_NO - (csl_no + sl_no + cgl_no + gl_no)
    if spare_links > 0:
      sl_no = min(sl_no + spare_links, len(self._secure_links))
      spare_links = MAX_SAME_DOMAIN_URLS_NO - (csl_no + sl_no + cgl_no + gl_no)
    if spare_links > 0:
      gl_no = min(gl_no + spare_links, len(self._general_links))

    for no_of_links, links in [
        (csl_no, self._clues_secure_links),
        (sl_no, self._secure_links),
        (cgl_no, self._clues_general_links),
        (gl_no, self._general_links)]:
      for i in xrange(no_of_links):
        if not links:
          break
        url = links.pop(0)
        self._links_visited.append(url)
        r = Retriever(url, self._domain, self._cookie_file)
        r.InitRequestHead()
        curl_multi_object.add_handle(r._curl_object)
        self._retrievers_list.append(r)

    if self._retrievers_list:
      try:
        self._MultiPerform(curl_multi_object)
      except pycurl.error as e:
        self.logger.error('Error: %s, url: %s', e, self._url)
      finally:
        for r in self._retrievers_list:
          curl_multi_object.remove_handle(r._curl_object)
      # |_retrievers_list[:]| is a copy of |_retrievers_list| to avoid removing
      # items from the iterated list.
      for r in self._retrievers_list[:]:
        r._url = urlparse.urljoin(r._url, r._curl_object.getinfo(
            pycurl.EFFECTIVE_URL))
        content_type = r._curl_object.getinfo(pycurl.CONTENT_TYPE)
        if content_type and ('text/html' in content_type.lower()):
          r.InitRequestGet()
          curl_multi_object.add_handle(r._curl_object)
        else:
          self._retrievers_list.remove(r)
          self.logger.info('\tSkipping: Not an HTML page <<< %s', r._url)
      if self._retrievers_list:
        try:
          self._MultiPerform(curl_multi_object)
        except pycurl.error as e:
          self.logger.error('Error: %s, url: %s', e, self._url)
        finally:
          for r in self._retrievers_list:
            curl_multi_object.remove_handle(r._curl_object)
            self.logger.info('Downloaded: %s', r._url)

  def _LogRegPageFound(self, retriever):
    """Display logging for registration page found.

    Args:
      retriever: The object that has retrieved the page.
    """
    self.logger.info('\t##############################################')
    self.logger.info('\t### %s ###', retriever._domain)
    self.logger.info('\t##############################################')
    self.logger.info('\t!!!!!!!!!  registration page FOUND !!!!!!!!!!!')
    self.logger.info('\t%s', retriever._url)
    self.logger.info('\t##############################################')

  def _GetNewLinks(self, retriever):
    """Appends new links discovered by each retriever to the appropriate lists.

    Links are copied to the links list of the crawler object, which holds all
    the links found from all retrievers that the crawler object created. The
    Crawler object exists as far as a specific site is examined and the
    Retriever object exists as far as a page of this site is examined.

    Args:
      retriever: a temporary object that downloads a specific page, parses the
          content and gets the page's href link.
    """
    for link in retriever._clues_secure_links:
      if (not link in self._clues_secure_links and
          not link in self._links_visited):
        self._clues_secure_links.append(link)
    for link in retriever._secure_links:
      if (not link in self._secure_links and
          not link in self._links_visited):
        self._secure_links.append(link)
    for link in retriever._clues_general_links:
      if (not link in self._clues_general_links and
          not link in self._links_visited):
        self._clues_general_links.append(link)
    for link in retriever._general_links:
      if (not link in self._general_links and
          not link in self._links_visited):
        self._general_links.append(link)

  def Run(self):
    """Runs the Crawler.

    Creates a Retriever object and calls its run method to get the first links,
    and then uses CurlMulti object and creates many Retriever objects to get
    the subsequent pages.

    The number of pages (=Retriever objs) created each time is restricted by
    MAX_SAME_DOMAIN_URLS_NO. After this number of Retriever objects download
    and parse their pages, we do the same again. The number of total pages
    visited is kept in urls_visited.
    If no registration page is found, the Crawler object will give up its try
    after MAX_TOTAL_URLS_PER_DOMAIN is reached.

    Returns:
      True is returned if registration page is found, or False otherwise.
    """
    reg_page_found = False
    if self.url_error:
      return False
    r = Retriever(self._url, self._domain, self._cookie_file)
    if r.Run():
      self._LogRegPageFound(r)
      reg_page_found = True
    else:
      self._url = r._url
      self._domain = r._domain
      self.logger.info('url to crawl: %s', self._url)
      self.logger.info('domain: %s', self._domain)
      self._links_visited.append(r._url)
      self._GetNewLinks(r)
      urls_visited = 1
      while True:
        if (not (self._clues_secure_links or self._secure_links or
                self._clues_general_links or self._general_links) or
            urls_visited >= MAX_TOTAL_URLS_PER_DOMAIN):
          break  # Registration page not found.
        m = pycurl.CurlMulti()
        self._GetLinksPages(m)
        urls_visited += len(self._retrievers_list)
        self.logger.info('\t<----- URLs visited for domain "%s": %d ----->',
                         self._domain, urls_visited)
        for r in self._retrievers_list:
          if r.ParseAndGetLinks():
            self._LogRegPageFound(r)
            reg_page_found = True
            break
          else:
            self.logger.info('parsed: %s', r._url)
            self._GetNewLinks(r)
        m.close()
        if reg_page_found:
          break
    while self._retrievers_list:
      r = self._retrievers_list.pop()
    return reg_page_found


class WorkerThread(threading.Thread):
  """Creates a new thread of execution."""
  def __init__(self, url):
    """Creates _url and page_found attri to populate urls_with_no_reg_page file.

    Used after thread's termination for the creation of a file with a list of
    the urls for which a registration page wasn't found.

    Args:
      url: will be used as an argument to create a Crawler object later.
    """
    threading.Thread.__init__(self)
    self._url = url
    self.page_found = False

  def run(self):
    """Execution of thread creates a Crawler object and runs it.

    Caution: this function name should not be changed to 'Run' or any other
    names because it is overriding the 'run' method of the 'threading.Thread'
    class. Otherwise it will never be called.
    """
    self.page_found = Crawler(self._url).Run()


class ThreadedCrawler(object):
  """Calls the Run function of WorkerThread which creates & runs a Crawler obj.

  The crawler object runs concurrently, examining one site each.
  """
  logger = logging.getLogger(__name__)

  def __init__(self, urls_file, logging_level=None):
    """Creates threaded Crawler objects.

    Args:
      urls_file: a text file containing a URL in each line.
      logging_level: verbosity level, default is None.

    Raises:
      IOError: If cannot find URLs from the list.
    """
    if logging_level:
      self.logger.setLevel(logging_level)

    self._urls_list = []
    f = open(urls_file)
    try:
      for url in f.readlines():
        url = url.strip()
        if not urlparse.urlparse(url)[0].startswith('http'):
          self.logger.info(
              '%s: skipping this (does not begin with "http://")', url)
          continue
        self._urls_list.append(url)
    except IOError as e:
      self.logger.error('Error: %s', e)
      raise
    finally:
      f.close()
    if not self._urls_list:
      error_msg = 'No URLs were found.'
      self.logger.error('ERROR: %s', error_msg)
      raise IOError(error_msg)

  def Run(self):
    """Runs Crawler objects using python threads.

    Number of concurrent threads is restricted to MAX_ALLOWED_THREADS.

    Returns:
      The number of registration pages found. -1 if no URLs are given.

    Raises:
      OSError: When creating the same directory that already exists.
    """
    if self._urls_list:
      allThreads = []
      # originalNumThreads is the number of threads just before the
      # ThreadedCrawler starts creating new threads. As a standalone script it
      # will be 1.
      originalNumThreads = threading.active_count()
      for url in self._urls_list:
        self.logger.info('URL fed to a crawler thread: %s', url)
        t = WorkerThread(url)
        t.start()
        allThreads.append(t)
        while threading.active_count() >= (
            MAX_ALLOWED_THREADS + originalNumThreads):
          time.sleep(.4)
      while threading.active_count() > originalNumThreads:
        time.sleep(.4)
      self.logger.info('----------------')
      self.logger.info('--- FINISHED ---')
      self.logger.info('----------------')
      urls_no = 0
      urls_not_found_no = 0
      not_file_name = os.path.join(
          REGISTER_PAGE_DIR, NOT_FOUND_REG_PAGE_SITES_FILENAME)
      not_file_dir = os.path.dirname(not_file_name)
      try:
        os.makedirs(not_file_dir)
      except OSError as e:
        if e.errno != errno.EEXIST:
          raise
      fnot = open(not_file_name, 'wb')
      try:
        for t in sorted(allThreads, key=lambda t: t._url):
          urls_no += 1
          if not t.page_found:
            urls_not_found_no += 1
            fnot.write('%s' % t._url)
            fnot.write(os.linesep)
      except IOError as e:
        self.logger.error('Error: %s', e)
      finally:
        fnot.close()
      self.logger.info('Total number of URLs given: %d\n', urls_no)
      self.logger.info(
          'Registration pages found: %d\n', (urls_no - urls_not_found_no))
      self.logger.info(
          'URLs that did not return a registration page: %d\n',
          urls_not_found_no)
      return urls_no - urls_not_found_no
    else:
      self.logger.error('Error: no URLs were found.')
      return -1


def main():
  usage = 'usage: %prog [options] single_url_or_urls_filename'
  parser = optparse.OptionParser(usage)
  parser.add_option(
      '-l', '--log_level', metavar='LOG_LEVEL', default='error',
      help='LOG_LEVEL: debug, info, warning or error [default: %default]')

  (options, args) = parser.parse_args()
  options.log_level = options.log_level.upper()
  if options.log_level not in ['DEBUG', 'INFO', 'WARNING', 'ERROR']:
    print 'Wrong log_level argument.'
    parser.print_help()
    return 1
  options.log_level = getattr(logging, options.log_level)

  if len(args) != 1:
    parser.error('Wrong number of arguments.')

  logger = logging.getLogger(__name__)
  if options.log_level:
    console = logging.StreamHandler()
    logger.addHandler(console)
    logger.setLevel(options.log_level)

  arg_is_a_file = os.path.isfile(args[0])
  if arg_is_a_file:
    CrawlerClass = ThreadedCrawler
  else:
    CrawlerClass = Crawler
  t0 = datetime.datetime.now()
  c = CrawlerClass(args[0], options.log_level)
  c.Run()
  if not arg_is_a_file and c.url_error:
    logger.error(
        'ERROR: "%s" is neither a valid filename nor a valid URL' % args[0])
  t1 = datetime.datetime.now()
  delta_t = t1 - t0
  logger.info('Started at: %s\n', t0)
  logger.info('Ended at: %s\n', t1)
  logger.info('Total execution time: %s\n', delta_t)
  return 0


if __name__ == "__main__":
  sys.exit(main())